From 06896124aa995f7c39d90094beb0cc10c89b6b43 Mon Sep 17 00:00:00 2001 From: Vlad Lazar Date: Thu, 21 Dec 2023 09:01:28 -0500 Subject: [PATCH 01/34] Zstd compressor (#3) Co-authored-by: Philippe Leprince Signed-off-by: Vlad Lazar * better multy-type compression * Version the Stream --------- Signed-off-by: Vlad Lazar Co-authored-by: Philippe Leprince --- .gitignore | 4 + src/bin/exrheader/main.cpp | 2 + src/examples/deepExamples.cpp | 89 +++++++----- src/examples/deepTiledExamples.cpp | 11 +- src/lib/OpenEXR/CMakeLists.txt | 6 + src/lib/OpenEXR/ImfCompression.h | 2 + src/lib/OpenEXR/ImfCompressionAttribute.cpp | 3 +- src/lib/OpenEXR/ImfCompressor.cpp | 13 +- src/lib/OpenEXR/ImfHeader.cpp | 14 ++ src/lib/OpenEXR/ImfHeader.h | 5 + src/lib/OpenEXR/ImfMultiPartInputFile.cpp | 1 + src/lib/OpenEXR/ImfZstdCompressor.cpp | 143 ++++++++++++++++++++ src/lib/OpenEXR/ImfZstdCompressor.h | 39 ++++++ src/lib/OpenEXRCore/base.c | 17 +++ src/lib/OpenEXRCore/openexr_base.h | 11 ++ 15 files changed, 321 insertions(+), 39 deletions(-) create mode 100644 src/lib/OpenEXR/ImfZstdCompressor.cpp create mode 100644 src/lib/OpenEXR/ImfZstdCompressor.h diff --git a/.gitignore b/.gitignore index db46b006ef..3669dd533e 100644 --- a/.gitignore +++ b/.gitignore @@ -57,3 +57,7 @@ docs/_test_images/ # Ignore Bazel generated files bazel-* + +cmake-build-debug/ + +.idea/ diff --git a/src/bin/exrheader/main.cpp b/src/bin/exrheader/main.cpp index 2f25edd44c..b1e9ecbb63 100644 --- a/src/bin/exrheader/main.cpp +++ b/src/bin/exrheader/main.cpp @@ -65,6 +65,8 @@ printCompression (Compression c) case DWAB_COMPRESSION: cout << "dwa, medium scanline blocks"; break; + case ZSTD_COMPRESSION: cout << "zstd"; break; + default: cout << int (c); break; } } diff --git a/src/examples/deepExamples.cpp b/src/examples/deepExamples.cpp index 9abd61d030..b28a0cf3bd 100644 --- a/src/examples/deepExamples.cpp +++ b/src/examples/deepExamples.cpp @@ -50,7 +50,7 @@ readDeepScanlineFile ( // - allocate the memory requred to store the samples // - read the pixels from the file // - + DeepScanLineInputFile file (filename); const Header& header = file.header (); @@ -118,20 +118,18 @@ readDeepScanlineFile ( } } -unsigned int getPixelSampleCount (int i, int j) +unsigned int +getPixelSampleCount (int i, int j) { // Dummy code creating deep data from a flat image return 1; } Array2D testDataZ; -Array2D testDataA; +Array2D testDataA; -void getPixelSampleData( - int i, - int j, - Array2D& dataZ, - Array2D& dataA) +void +getPixelSampleData (int i, int j, Array2D& dataZ, Array2D& dataA) { // Dummy code creating deep data from a flat image dataZ[i][j][0] = testDataZ[i][j]; @@ -147,7 +145,8 @@ writeDeepScanlineFile ( Array2D& dataA, - Array2D& sampleCount) + Array2D& sampleCount, + Compression compression = Compression::ZIPS_COMPRESSION) { // @@ -160,7 +159,7 @@ writeDeepScanlineFile ( // - describe the memory layout of the A and Z pixels // - store the pixels in the file // - + int height = dataWindow.max.y - dataWindow.min.y + 1; int width = dataWindow.max.x - dataWindow.min.x + 1; @@ -169,7 +168,7 @@ writeDeepScanlineFile ( header.channels ().insert ("Z", Channel (FLOAT)); header.channels ().insert ("A", Channel (HALF)); header.setType (DEEPSCANLINE); - header.compression () = ZIPS_COMPRESSION; + header.compression () = compression; DeepScanLineOutputFile file (filename, header); @@ -211,7 +210,7 @@ writeDeepScanlineFile ( dataZ[i][j] = new float[sampleCount[i][j]]; dataA[i][j] = new half[sampleCount[i][j]]; // Generate data for dataZ and dataA. - getPixelSampleData(i, j, dataZ, dataA); + getPixelSampleData (i, j, dataZ, dataA); } file.writePixels (1); @@ -227,30 +226,56 @@ writeDeepScanlineFile ( } } - -void deepExamples() +void +deepExamples () { int w = 800; int h = 600; - + Box2i window; - window.min.setValue(0, 0); - window.max.setValue(w - 1, h - 1); - - Array2D dataZ; - dataZ.resizeErase(h, w); - - Array2D dataA; - dataA.resizeErase(h, w); - + window.min.setValue (0, 0); + window.max.setValue (w - 1, h - 1); + + Array2D dataZ; + dataZ.resizeErase (h, w); + + Array2D dataA; + dataA.resizeErase (h, w); + Array2D sampleCount; - sampleCount.resizeErase(h, w); - + sampleCount.resizeErase (h, w); + // Create an image to be used as a source for deep data - testDataA.resizeErase(h, w); - testDataZ.resizeErase(h, w); - drawImage2(testDataA, testDataZ, w, h); - - writeDeepScanlineFile("test.deep.exr", window, window, dataZ, dataA, sampleCount); - readDeepScanlineFile ("test.deep.exr", window, window, dataZ, dataA, sampleCount); + testDataA.resizeErase (h, w); + testDataZ.resizeErase (h, w); + drawImage2 (testDataA, testDataZ, w, h); + + { + writeDeepScanlineFile ( + "test.deep.exr", + window, + window, + dataZ, + dataA, + sampleCount, + Compression::ZSTD_COMPRESSION); + } + { + writeDeepScanlineFile ( + "test.zips.exr", + window, + window, + dataZ, + dataA, + sampleCount, + Compression::ZIPS_COMPRESSION); + } + { + readDeepScanlineFile ( + "test.deep.exr", window, window, dataZ, dataA, sampleCount); + } + { + readDeepScanlineFile ( + "test.zips.exr", window, window, dataZ, dataA, sampleCount); + } } diff --git a/src/examples/deepTiledExamples.cpp b/src/examples/deepTiledExamples.cpp index d6ec256361..3e58600c09 100644 --- a/src/examples/deepTiledExamples.cpp +++ b/src/examples/deepTiledExamples.cpp @@ -163,7 +163,8 @@ writeDeepTiledFile ( Box2i displayWindow, Box2i dataWindow, int tileSizeX, - int tileSizeY) + int tileSizeY, + Compression compression = Compression::ZIPS_COMPRESSION) { // // Write a deep image with only a A (alpha) and a Z (depth) channel, @@ -183,7 +184,7 @@ writeDeepTiledFile ( header.channels ().insert ("Z", Channel (FLOAT)); header.channels ().insert ("A", Channel (HALF)); header.setType (DEEPTILE); - header.compression () = ZIPS_COMPRESSION; + header.compression () = compression; header.setTileDescription ( TileDescription (tileSizeX, tileSizeY, ONE_LEVEL)); @@ -273,6 +274,8 @@ void deepTiledExamples() testDataZ.resizeErase(h, w); drawImage2(testDataA, testDataZ, w, h); - writeDeepTiledFile("testTiled.deep.exr", window, window, tileSizeX, tileSizeY); - readDeepTiledFile ("testTiled.deep.exr", window, window, dataZ, dataA, sampleCount); + writeDeepTiledFile("testTiled.deep.zip.exr", window, window, tileSizeX, tileSizeY); + readDeepTiledFile ("testTiled.deep.zip.exr", window, window, dataZ, dataA, sampleCount); + writeDeepTiledFile("testTiled.deep.zstd.exr", window, window, tileSizeX, tileSizeY, Compression::ZSTD_COMPRESSION); + readDeepTiledFile ("testTiled.deep.zstd.exr", window, window, dataZ, dataA, sampleCount); } diff --git a/src/lib/OpenEXR/CMakeLists.txt b/src/lib/OpenEXR/CMakeLists.txt index b58e0bace1..45a809f386 100644 --- a/src/lib/OpenEXR/CMakeLists.txt +++ b/src/lib/OpenEXR/CMakeLists.txt @@ -29,6 +29,7 @@ openexr_define_library(OpenEXR ImfTiledMisc.h ImfZip.h ImfZipCompressor.h + ImfZstdCompressor.h b44ExpLogTable.h dwaLookups.h ImfAcesFile.cpp @@ -122,6 +123,7 @@ openexr_define_library(OpenEXR ImfWav.cpp ImfZip.cpp ImfZipCompressor.cpp + ImfZstdCompressor.cpp HEADERS ImfAcesFile.h ImfArray.h @@ -220,3 +222,7 @@ openexr_define_library(OpenEXR OpenEXR::IlmThread OpenEXR::OpenEXRCore ) + +target_include_directories(OpenEXR PUBLIC "/home/vladal/bin/include/") +target_link_directories(OpenEXR PUBLIC "/home/vladal/bin/lib") +target_link_libraries(OpenEXR PUBLIC "dl" "blosc2") \ No newline at end of file diff --git a/src/lib/OpenEXR/ImfCompression.h b/src/lib/OpenEXR/ImfCompression.h index fbc0fa24ab..e8f3386bf9 100644 --- a/src/lib/OpenEXR/ImfCompression.h +++ b/src/lib/OpenEXR/ImfCompression.h @@ -45,6 +45,8 @@ enum IMF_EXPORT_ENUM Compression // wise and faster to decode full frames // than DWAA_COMPRESSION. + ZSTD_COMPRESSION = 10, + NUM_COMPRESSION_METHODS // number of different compression methods }; diff --git a/src/lib/OpenEXR/ImfCompressionAttribute.cpp b/src/lib/OpenEXR/ImfCompressionAttribute.cpp index b0e93db479..a443eba8bb 100644 --- a/src/lib/OpenEXR/ImfCompressionAttribute.cpp +++ b/src/lib/OpenEXR/ImfCompressionAttribute.cpp @@ -57,7 +57,8 @@ CompressionAttribute::readValueFrom ( tmp != ZIPS_COMPRESSION && tmp != ZIP_COMPRESSION && tmp != PIZ_COMPRESSION && tmp != PXR24_COMPRESSION && tmp != B44_COMPRESSION && tmp != B44A_COMPRESSION && - tmp != DWAA_COMPRESSION && tmp != DWAB_COMPRESSION) + tmp != DWAA_COMPRESSION && tmp != DWAB_COMPRESSION && + tmp != ZSTD_COMPRESSION) { tmp = NUM_COMPRESSION_METHODS; } diff --git a/src/lib/OpenEXR/ImfCompressor.cpp b/src/lib/OpenEXR/ImfCompressor.cpp index 4d9e99a33a..dfb5ab8adf 100644 --- a/src/lib/OpenEXR/ImfCompressor.cpp +++ b/src/lib/OpenEXR/ImfCompressor.cpp @@ -18,6 +18,7 @@ #include "ImfPxr24Compressor.h" #include "ImfRleCompressor.h" #include "ImfZipCompressor.h" +#include "ImfZstdCompressor.h" OPENEXR_IMF_INTERNAL_NAMESPACE_SOURCE_ENTER @@ -63,7 +64,8 @@ isValidCompression (Compression c) case B44_COMPRESSION: case B44A_COMPRESSION: case DWAA_COMPRESSION: - case DWAB_COMPRESSION: return true; + case DWAB_COMPRESSION: + case ZSTD_COMPRESSION: return true; default: return false; } @@ -89,7 +91,8 @@ isValidDeepCompression (Compression c) { case NO_COMPRESSION: case RLE_COMPRESSION: - case ZIPS_COMPRESSION: return true; + case ZIPS_COMPRESSION: + case ZSTD_COMPRESSION: return true; default: return false; } } @@ -141,6 +144,8 @@ newCompressor (Compression c, size_t maxScanLineSize, const Header& hdr) 256, DwaCompressor::STATIC_HUFFMAN); + case ZSTD_COMPRESSION: + return new ZstdCompressor (hdr, maxScanLineSize, 32); default: return 0; } } @@ -162,6 +167,7 @@ numLinesInBuffer (Compression comp) case B44_COMPRESSION: case B44A_COMPRESSION: case DWAA_COMPRESSION: return 32; + case ZSTD_COMPRESSION: return 32; case DWAB_COMPRESSION: return 256; default: throw IEX_NAMESPACE::ArgExc ("Unknown compression type"); @@ -182,6 +188,9 @@ newTileCompressor ( case ZIP_COMPRESSION: return new ZipCompressor (hdr, tileLineSize, numTileLines); + case ZSTD_COMPRESSION: + + return new ZstdCompressor (hdr, tileLineSize, numTileLines); case PIZ_COMPRESSION: diff --git a/src/lib/OpenEXR/ImfHeader.cpp b/src/lib/OpenEXR/ImfHeader.cpp index be7f3c9664..cebec77aca 100644 --- a/src/lib/OpenEXR/ImfHeader.cpp +++ b/src/lib/OpenEXR/ImfHeader.cpp @@ -70,9 +70,11 @@ struct CompressionRecord { exr_get_default_zip_compression_level(&zip_level); exr_get_default_dwa_compression_quality(&dwa_level); + exr_get_default_zstd_compression_level(&zstd_level); } int zip_level; float dwa_level; + int zstd_level; }; // NB: This is extra complicated than one would normally write to // handle scenario that seems to happen on MacOS/Windows (probably @@ -696,6 +698,18 @@ Header::zipCompressionLevel () const return retrieveCompressionRecord (this).zip_level; } +int& +Header::zstdCompressionLevel () +{ + return retrieveCompressionRecord (this).zstd_level; +} + +int +Header::zstdCompressionLevel () const +{ + return retrieveCompressionRecord (this).zstd_level; +} + float& Header::dwaCompressionLevel () { diff --git a/src/lib/OpenEXR/ImfHeader.h b/src/lib/OpenEXR/ImfHeader.h index e09782b89c..4aab3a1b64 100644 --- a/src/lib/OpenEXR/ImfHeader.h +++ b/src/lib/OpenEXR/ImfHeader.h @@ -285,6 +285,11 @@ class IMF_EXPORT_TYPE Header float& dwaCompressionLevel (); IMF_EXPORT float dwaCompressionLevel () const; + IMF_EXPORT + int& zstdCompressionLevel (); + IMF_EXPORT + int zstdCompressionLevel () const; + IMF_EXPORT //----------------------------------------------------- // Access to required attributes for multipart files diff --git a/src/lib/OpenEXR/ImfMultiPartInputFile.cpp b/src/lib/OpenEXR/ImfMultiPartInputFile.cpp index a5c17e3781..b9ed6d5d29 100644 --- a/src/lib/OpenEXR/ImfMultiPartInputFile.cpp +++ b/src/lib/OpenEXR/ImfMultiPartInputFile.cpp @@ -547,6 +547,7 @@ MultiPartInputFile::Data::chunkOffsetReconstruction ( // (TODO) fix this so that it doesn't need to be revised for future compression types. switch (parts[i]->header.compression ()) { + case ZSTD_COMPRESSION: rowsizes[i] = 32; break; case DWAB_COMPRESSION: rowsizes[i] = 256; break; case PIZ_COMPRESSION: case B44_COMPRESSION: diff --git a/src/lib/OpenEXR/ImfZstdCompressor.cpp b/src/lib/OpenEXR/ImfZstdCompressor.cpp new file mode 100644 index 0000000000..43fa4c640c --- /dev/null +++ b/src/lib/OpenEXR/ImfZstdCompressor.cpp @@ -0,0 +1,143 @@ +#include +#include "ImfZstdCompressor.h" + +#include "blosc2.h" +#include "IlmThreadPool.h" +#include "ImfChannelList.h" +#include "ImfMisc.h" +namespace +{ +class BloscInit +{ +public: + static void Init () { getInstance (); } + BloscInit (const BloscInit&) = delete; + BloscInit& operator= (const BloscInit&) = delete; + +private: + BloscInit () { blosc2_init (); } + ~BloscInit () { blosc2_destroy (); } + static BloscInit& getInstance () + { + static BloscInit instance; + return instance; + } +}; +} // namespace + +OPENEXR_IMF_INTERNAL_NAMESPACE_SOURCE_ENTER +ZstdCompressor::ZstdCompressor ( + const Header& hdr, size_t maxScanlineSize, size_t numScanLines) + : Compressor (hdr) + , _maxScanlineSize (maxScanlineSize) + , _numScanLines (numScanLines) + , _outBuffer (nullptr, &free) + , _schunk (nullptr, &blosc2_schunk_free) +{} + +int +ZstdCompressor::numScanLines () const +{ + return _numScanLines; // Needs to be in sync with ImfCompressor::numLinesInBuffer +} + +int +ZstdCompressor::compress ( + const char* inPtr, int inSize, int minY, const char*& outPtr) +{ + int typeSize = std::numeric_limits::min (); + for (auto it = header ().channels ().begin (); + it != header ().channels ().end (); + ++it) + { + // BLOSC prefilter is affected by the typesize. Initializing to max will ensure that a channel is not split in 2 and filtered separately. + // probably compression can be improved for non-deep images by compressing every channel separately with the correct typeSize + // (much harder to do for Deep-Data). + typeSize = std::max (typeSize, Imf::pixelTypeSize (it.channel ().type)); + } + + auto ret = BLOSC_compress_impl (inPtr, inSize, typeSize, outPtr); + auto data = malloc (Xdr::size () + ret); + auto write = (char*) data; + + Xdr::write (write, Versions::LATEST); + + memcpy (write, outPtr, ret); + outPtr = (char*) data; + + _outBuffer = raw_ptr ((char*) data, &free); + + return ret; +} + +int +ZstdCompressor::uncompress ( + const char* inPtr, int inSize, int minY, const char*& outPtr) +{ + auto read = (const char*) inPtr; + int v; + Xdr::read (read, v); + if (v == Versions::SINGLE_BLOB) + { + return BLOSC_uncompress_impl_single_blob ( + read, inSize - Xdr::size (), outPtr); + } + else + { + throw Iex::InputExc ("Unsupported ZstdCompressor version"); + } +} + +int +ZstdCompressor::BLOSC_compress_impl ( + const char* inPtr, int inSize, int typeSize, const char*& out) +{ + BloscInit::Init (); + blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS; + + cparams.typesize = typeSize; + // clevel 9 is about a 20% increase in compression compared to 5. + // Decompression speed is unchanged. + cparams.clevel = header ().zstdCompressionLevel (); + cparams.nthreads = 1; + cparams.compcode = BLOSC_ZSTD; // Codec + cparams.splitmode = + BLOSC_NEVER_SPLIT; // Split => multithreading, not split better compression + + blosc2_storage storage = BLOSC2_STORAGE_DEFAULTS; + storage.cparams = &cparams; + storage.contiguous = true; + + _schunk = schunk_ptr (blosc2_schunk_new (&storage), &blosc2_schunk_free); + + auto in = const_cast (inPtr); + blosc2_schunk_append_buffer (_schunk.get (), in, inSize); + + uint8_t* buffer; + bool shouldFree = true; + auto size = blosc2_schunk_to_buffer (_schunk.get (), &buffer, &shouldFree); + out = (char*) buffer; + if (shouldFree) { _outBuffer = raw_ptr ((char*) buffer, &free); } + return size; +} + +int +ZstdCompressor::BLOSC_uncompress_impl_single_blob ( + const char* inPtr, int inSize, const char*& out) +{ + auto in = const_cast (inPtr); + _schunk = schunk_ptr ( + blosc2_schunk_from_buffer ( + reinterpret_cast (in), inSize, true), + &blosc2_schunk_free); + + auto buffSize = _maxScanlineSize * numScanLines (); + _outBuffer = + Imf::ZstdCompressor::raw_ptr ((char*) malloc (buffSize), &free); + auto size = blosc2_schunk_decompress_chunk ( + _schunk.get (), 0, _outBuffer.get (), buffSize); + out = _outBuffer.get (); + return size; +} + +OPENEXR_IMF_INTERNAL_NAMESPACE_SOURCE_EXIT \ No newline at end of file diff --git a/src/lib/OpenEXR/ImfZstdCompressor.h b/src/lib/OpenEXR/ImfZstdCompressor.h new file mode 100644 index 0000000000..f1c9110dd0 --- /dev/null +++ b/src/lib/OpenEXR/ImfZstdCompressor.h @@ -0,0 +1,39 @@ +#pragma once +#include +#include "ImfNamespace.h" +#include "ImfCompressor.h" +#include "ImfHeader.h" +#include "blosc2.h" + +OPENEXR_IMF_INTERNAL_NAMESPACE_HEADER_ENTER +class ZstdCompressor : public Compressor +{ +public: + ZstdCompressor ( + const Header& hdr, size_t maxScanLines, size_t numScanLines); + +private: + using schunk_ptr = + std::unique_ptr; + using raw_ptr = std::unique_ptr; + raw_ptr _outBuffer; + schunk_ptr _schunk; + size_t _maxScanlineSize; + size_t _numScanLines; + int numScanLines () const override; // max + int compress ( + const char* inPtr, int inSize, int minY, const char*& outPtr) override; + int uncompress ( + const char* inPtr, int inSize, int minY, const char*& outPtr) override; + int BLOSC_compress_impl ( + const char* inPtr, int inSize, int typeSize, const char*& out); + int BLOSC_uncompress_impl_single_blob ( + const char* inPtr, int inSize, const char*& out); + enum Versions : int + { + SINGLE_BLOB = 1, + LATEST = SINGLE_BLOB + }; +}; + +OPENEXR_IMF_INTERNAL_NAMESPACE_HEADER_EXIT \ No newline at end of file diff --git a/src/lib/OpenEXRCore/base.c b/src/lib/OpenEXRCore/base.c index 513e73ea2c..8659a6b4fb 100644 --- a/src/lib/OpenEXRCore/base.c +++ b/src/lib/OpenEXRCore/base.c @@ -208,3 +208,20 @@ exr_get_default_dwa_compression_quality (float* q) { if (q) *q = sDefaultDwaLevel; } + +// 9 is 20% more expensive to compress. Decompression time remains constant. +static int sDefaultZstdLevel = 5; + +void +exr_set_default_zstd_compression_level (int q) +{ + if (q < 0) q = 0; + if (q > 9) q = 9; + sDefaultZstdLevel = q; +} + +void +exr_get_default_zstd_compression_level (int* q) +{ + if (q) *q = sDefaultZstdLevel; +} \ No newline at end of file diff --git a/src/lib/OpenEXRCore/openexr_base.h b/src/lib/OpenEXRCore/openexr_base.h index 8df304235b..a145323aa2 100644 --- a/src/lib/OpenEXRCore/openexr_base.h +++ b/src/lib/OpenEXRCore/openexr_base.h @@ -136,6 +136,17 @@ EXR_EXPORT void exr_set_default_dwa_compression_quality (float q); */ EXR_EXPORT void exr_get_default_dwa_compression_quality (float* q); +/** @brief Assigns a default zstd compression level. + * + * This value may be controlled separately on each part, but this + * global control determines the initial value. + */ +EXR_EXPORT void exr_set_default_zstd_compression_level (int l); + +/** @brief Retrieve the global default zstd compression value + */ +EXR_EXPORT void exr_get_default_zstd_compression_level (int* l); + /** @} */ /** From 574988ee7d43ed764d7facc450a91e3300b19431 Mon Sep 17 00:00:00 2001 From: Vlad-Andrei Lazar Date: Mon, 8 Jan 2024 15:24:10 -0500 Subject: [PATCH 02/34] Fix single blob --- src/examples/deepTiledExamples.cpp | 147 ++++++++++++++------------ src/lib/OpenEXR/ImfZstdCompressor.cpp | 14 ++- 2 files changed, 83 insertions(+), 78 deletions(-) diff --git a/src/examples/deepTiledExamples.cpp b/src/examples/deepTiledExamples.cpp index 3e58600c09..4f2d22f3c2 100644 --- a/src/examples/deepTiledExamples.cpp +++ b/src/examples/deepTiledExamples.cpp @@ -28,17 +28,12 @@ using namespace IMF; using namespace std; using namespace IMATH_NAMESPACE; - // defined in deepExamples.cpp extern Array2D testDataZ; -extern Array2D testDataA; -extern unsigned int getPixelSampleCount (int i, int j); -extern void getPixelSampleData( - int i, - int j, - Array2D& dataZ, - Array2D& dataA); - +extern Array2D testDataA; +extern unsigned int getPixelSampleCount (int i, int j); +extern void getPixelSampleData ( + int i, int j, Array2D& dataZ, Array2D& dataA); void readDeepTiledFile ( @@ -62,24 +57,24 @@ readDeepTiledFile ( // - allocate the memory requred to store the samples // - read the pixels from the file // - + DeepTiledInputFile file (filename); - + int width = dataWindow.max.x - dataWindow.min.x + 1; int height = dataWindow.max.y - dataWindow.min.y + 1; - + sampleCount.resizeErase (height, width); dataZ.resizeErase (height, width); dataA.resizeErase (height, width); - + DeepFrameBuffer frameBuffer; - + frameBuffer.insertSampleCountSlice (Slice ( UINT, (char*) (&sampleCount[0][0] - dataWindow.min.x - dataWindow.min.y * width), sizeof (unsigned int) * 1, // xStride sizeof (unsigned int) * width)); // yStride - + frameBuffer.insert ( "Z", DeepSlice ( @@ -88,7 +83,7 @@ readDeepTiledFile ( sizeof (float*) * 1, // xStride for pointer array sizeof (float*) * width, // yStride for pointer array sizeof (float) * 1)); // stride for samples - + frameBuffer.insert ( "A", DeepSlice ( @@ -97,14 +92,14 @@ readDeepTiledFile ( sizeof (half*) * 1, // xStride for pointer array sizeof (half*) * width, // yStride for pointer array sizeof (half) * 1)); // stride for samples - + file.setFrameBuffer (frameBuffer); - + int numXTiles = file.numXTiles (0); int numYTiles = file.numYTiles (0); - + file.readPixelSampleCounts (0, numXTiles - 1, 0, numYTiles - 1); - + for (int i = 0; i < height; i++) { for (int j = 0; j < width; j++) @@ -113,11 +108,11 @@ readDeepTiledFile ( dataA[i][j] = new half[sampleCount[i][j]]; } } - + file.readTiles (0, numXTiles - 1, 0, numYTiles - 1); - + // (after read data is processed, data must be freed:) - + for (int i = 0; i < height; i++) { for (int j = 0; j < width; j++) @@ -128,42 +123,43 @@ readDeepTiledFile ( } } -void getSampleDataForTile ( - int i, - int j, - int tileSizeX, - int tileSizeY, - Array2D& sampleCount, - Array2D& dataZ, - Array2D& dataA) +void +getSampleDataForTile ( + int i, + int j, + int tileSizeX, + int tileSizeY, + Array2D& sampleCount, + Array2D& dataZ, + Array2D& dataA) { for (int k = 0; k < tileSizeY; k++) { int y = j * tileSizeY + k; - if (y >= sampleCount.height()) break; - + if (y >= sampleCount.height ()) break; + for (int l = 0; l < tileSizeX; l++) { int x = i * tileSizeX + l; - if (x >= sampleCount.width()) break; - - sampleCount[y][x] = getPixelSampleCount(y, x); - + if (x >= sampleCount.width ()) break; + + sampleCount[y][x] = getPixelSampleCount (y, x); + dataZ[y][x] = new float[sampleCount[y][x]]; - dataA[y][x] = new half [sampleCount[y][x]]; - - getPixelSampleData(y, x, dataZ, dataA); + dataA[y][x] = new half[sampleCount[y][x]]; + + getPixelSampleData (y, x, dataZ, dataA); } } } void writeDeepTiledFile ( - const char filename[], - Box2i displayWindow, - Box2i dataWindow, - int tileSizeX, - int tileSizeY, + const char filename[], + Box2i displayWindow, + Box2i dataWindow, + int tileSizeX, + int tileSizeY, Compression compression = Compression::ZIPS_COMPRESSION) { // @@ -176,7 +172,7 @@ writeDeepTiledFile ( // - describe the memory layout of the A and Z pixels // - store the pixels in the file // - + int height = dataWindow.max.y - dataWindow.min.y + 1; int width = dataWindow.max.x - dataWindow.min.x + 1; @@ -233,11 +229,12 @@ writeDeepTiledFile ( for (int i = 0; i < file.numXTiles (0); i++) { // Generate data for sampleCount, dataZ and dataA. - getSampleDataForTile (i, j, tileSizeX, tileSizeY, sampleCount, dataZ, dataA); + getSampleDataForTile ( + i, j, tileSizeX, tileSizeY, sampleCount, dataZ, dataA); file.writeTile (i, j, 0); } } - + for (int i = 0; i < height; i++) { for (int j = 0; j < width; j++) @@ -248,34 +245,44 @@ writeDeepTiledFile ( } } -void deepTiledExamples() +void +deepTiledExamples () { int w = 800; int h = 600; - + int tileSizeX = 64; int tileSizeY = 64; - + Box2i window; - window.min.setValue(0, 0); - window.max.setValue(w - 1, h - 1); - - Array2D dataZ; - dataZ.resizeErase(h, w); - - Array2D dataA; - dataA.resizeErase(h, w); - + window.min.setValue (0, 0); + window.max.setValue (w - 1, h - 1); + + Array2D dataZ; + dataZ.resizeErase (h, w); + + Array2D dataA; + dataA.resizeErase (h, w); + Array2D sampleCount; - sampleCount.resizeErase(h, w); - + sampleCount.resizeErase (h, w); + // Create an image to be used as a source for deep data - testDataA.resizeErase(h, w); - testDataZ.resizeErase(h, w); - drawImage2(testDataA, testDataZ, w, h); - - writeDeepTiledFile("testTiled.deep.zip.exr", window, window, tileSizeX, tileSizeY); - readDeepTiledFile ("testTiled.deep.zip.exr", window, window, dataZ, dataA, sampleCount); - writeDeepTiledFile("testTiled.deep.zstd.exr", window, window, tileSizeX, tileSizeY, Compression::ZSTD_COMPRESSION); - readDeepTiledFile ("testTiled.deep.zstd.exr", window, window, dataZ, dataA, sampleCount); + testDataA.resizeErase (h, w); + testDataZ.resizeErase (h, w); + drawImage2 (testDataA, testDataZ, w, h); + + writeDeepTiledFile ( + "testTiled.deep.exr", window, window, tileSizeX, tileSizeY); + readDeepTiledFile ( + "testTiled.deep.exr", window, window, dataZ, dataA, sampleCount); + writeDeepTiledFile ( + "testTiled.deep.zstd.exr", + window, + window, + tileSizeX, + tileSizeY, + Compression::ZSTD_COMPRESSION); + readDeepTiledFile ( + "testTiled.deep.zstd.exr", window, window, dataZ, dataA, sampleCount); } diff --git a/src/lib/OpenEXR/ImfZstdCompressor.cpp b/src/lib/OpenEXR/ImfZstdCompressor.cpp index 43fa4c640c..48e742d591 100644 --- a/src/lib/OpenEXR/ImfZstdCompressor.cpp +++ b/src/lib/OpenEXR/ImfZstdCompressor.cpp @@ -56,9 +56,10 @@ ZstdCompressor::compress ( typeSize = std::max (typeSize, Imf::pixelTypeSize (it.channel ().type)); } - auto ret = BLOSC_compress_impl (inPtr, inSize, typeSize, outPtr); - auto data = malloc (Xdr::size () + ret); - auto write = (char*) data; + auto ret = BLOSC_compress_impl (inPtr, inSize, typeSize, outPtr); + auto fullSize = Xdr::size () + ret; + auto data = malloc (fullSize); + auto write = (char*) data; Xdr::write (write, Versions::LATEST); @@ -67,7 +68,7 @@ ZstdCompressor::compress ( _outBuffer = raw_ptr ((char*) data, &free); - return ret; + return fullSize; } int @@ -82,10 +83,7 @@ ZstdCompressor::uncompress ( return BLOSC_uncompress_impl_single_blob ( read, inSize - Xdr::size (), outPtr); } - else - { - throw Iex::InputExc ("Unsupported ZstdCompressor version"); - } + else { throw Iex::InputExc ("Unsupported ZstdCompressor version"); } } int From 0e90ff164e7da0fc6e739ba50b1e2c13a8deb471 Mon Sep 17 00:00:00 2001 From: Philippe Leprince Date: Wed, 10 Jan 2024 17:19:28 +0100 Subject: [PATCH 03/34] Add license headers. Clang-format example file. Clang-format deepExamples.cpp and fix a comment typo. Clang-format deepExamples.cpp and fix a comment typo. --- src/examples/deepExamples.cpp | 20 +++++++++++++------- src/lib/OpenEXR/ImfZstdCompressor.cpp | 5 +++++ src/lib/OpenEXR/ImfZstdCompressor.h | 6 ++++++ 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/src/examples/deepExamples.cpp b/src/examples/deepExamples.cpp index b28a0cf3bd..f7599084d5 100644 --- a/src/examples/deepExamples.cpp +++ b/src/examples/deepExamples.cpp @@ -47,7 +47,7 @@ readDeepScanlineFile ( // - allocate memory for the pixels // - describe the layout of the A, and Z pixel buffers // - read the sample counts from the file - // - allocate the memory requred to store the samples + // - allocate the memory required to store the samples // - read the pixels from the file // @@ -69,7 +69,8 @@ readDeepScanlineFile ( frameBuffer.insertSampleCountSlice (Slice ( UINT, - (char*) (&sampleCount[0][0] - dataWindow.min.x - dataWindow.min.y * width), + (char*) (&sampleCount[0][0] - dataWindow.min.x - + dataWindow.min.y * width), sizeof (unsigned int) * 1, // xStride sizeof (unsigned int) * width)); // yStride @@ -77,7 +78,8 @@ readDeepScanlineFile ( "dataZ", DeepSlice ( FLOAT, - (char*) (&dataZ[0][0] - dataWindow.min.x - dataWindow.min.y * width), + (char*) (&dataZ[0][0] - dataWindow.min.x - + dataWindow.min.y * width), sizeof (float*) * 1, // xStride for pointer array sizeof (float*) * width, // yStride for pointer array @@ -87,7 +89,8 @@ readDeepScanlineFile ( "dataA", DeepSlice ( HALF, - (char*) (&dataA[0][0] - dataWindow.min.x - dataWindow.min.y * width), + (char*) (&dataA[0][0] - dataWindow.min.x - + dataWindow.min.y * width), sizeof (half*) * 1, // xStride for pointer array sizeof (half*) * width, // yStride for pointer array sizeof (half) * 1)); // stride for O data sample @@ -176,7 +179,8 @@ writeDeepScanlineFile ( frameBuffer.insertSampleCountSlice (Slice ( UINT, - (char*) (&sampleCount[0][0] - dataWindow.min.x - dataWindow.min.y * width), + (char*) (&sampleCount[0][0] - dataWindow.min.x - + dataWindow.min.y * width), sizeof (unsigned int) * 1, // xS sizeof (unsigned int) * width)); // yStride @@ -185,7 +189,8 @@ writeDeepScanlineFile ( "Z", DeepSlice ( FLOAT, - (char*) (&dataZ[0][0] - dataWindow.min.x - dataWindow.min.y * width), + (char*) (&dataZ[0][0] - dataWindow.min.x - + dataWindow.min.y * width), sizeof (float*) * 1, // xStride for pointer sizeof (float*) * width, // yStride for pointer array @@ -195,7 +200,8 @@ writeDeepScanlineFile ( "A", DeepSlice ( HALF, - (char*) (&dataA[0][0] - dataWindow.min.x - dataWindow.min.y * width), + (char*) (&dataA[0][0] - dataWindow.min.x - + dataWindow.min.y * width), sizeof (half*) * 1, // xStride for pointer array sizeof (half*) * width, // yStride for pointer array sizeof (half) * 1)); // stride for A data sample diff --git a/src/lib/OpenEXR/ImfZstdCompressor.cpp b/src/lib/OpenEXR/ImfZstdCompressor.cpp index 48e742d591..d35a3deecd 100644 --- a/src/lib/OpenEXR/ImfZstdCompressor.cpp +++ b/src/lib/OpenEXR/ImfZstdCompressor.cpp @@ -1,3 +1,8 @@ +// +// SPDX-License-Identifier: BSD-3-Clause +// Copyright (c) Contributors to the OpenEXR Project. +// + #include #include "ImfZstdCompressor.h" diff --git a/src/lib/OpenEXR/ImfZstdCompressor.h b/src/lib/OpenEXR/ImfZstdCompressor.h index f1c9110dd0..d9319e076c 100644 --- a/src/lib/OpenEXR/ImfZstdCompressor.h +++ b/src/lib/OpenEXR/ImfZstdCompressor.h @@ -1,4 +1,10 @@ #pragma once + +// +// SPDX-License-Identifier: BSD-3-Clause +// Copyright (c) Contributors to the OpenEXR Project. +// + #include #include "ImfNamespace.h" #include "ImfCompressor.h" From 057dd80ab65fc03a0df375ddaec8be2d9d79b12e Mon Sep 17 00:00:00 2001 From: Vlad Lazar Date: Mon, 15 Jan 2024 10:15:10 -0500 Subject: [PATCH 04/34] Build Blosc as a part of the regular build (#4) --- cmake/LibraryDefine.cmake | 11 +++++ cmake/OpenEXRSetup.cmake | 88 ++++++++++++++++++++++++++++++++++ src/lib/OpenEXR/CMakeLists.txt | 5 +- 3 files changed, 100 insertions(+), 4 deletions(-) diff --git a/cmake/LibraryDefine.cmake b/cmake/LibraryDefine.cmake index a242ec8519..8df620ca30 100644 --- a/cmake/LibraryDefine.cmake +++ b/cmake/LibraryDefine.cmake @@ -30,6 +30,17 @@ function(OPENEXR_DEFINE_LIBRARY libname) # we are embedding libdeflate target_include_directories(${objlib} PRIVATE ${EXR_DEFLATE_INCLUDE_DIR}) + # we are statically linking blosc2 + if(${objlib} STREQUAL "OpenEXR") # OR ${objlib} STREQUAL "OpenEXRCore" + target_include_directories(${objlib} PRIVATE ${BLOSC2_INCLUDE_DIRS}) + target_link_directories(${objlib} PRIVATE ${BLOSC2_LIB_DIR}) + target_link_libraries(${objlib} PRIVATE "dl" ${BLOSC2_LIB_NAME}) + if(TARGET blosc2_static) + install(TARGETS blosc2_static EXPORT ${objlib}) + endif() + endif() + + if(OPENEXR_CURLIB_PRIV_EXPORT AND BUILD_SHARED_LIBS) target_compile_definitions(${objlib} PRIVATE ${OPENEXR_CURLIB_PRIV_EXPORT}) if(WIN32) diff --git a/cmake/OpenEXRSetup.cmake b/cmake/OpenEXRSetup.cmake index 7d01362c84..c0c15174b0 100644 --- a/cmake/OpenEXRSetup.cmake +++ b/cmake/OpenEXRSetup.cmake @@ -290,6 +290,94 @@ else() endif() endif() +####################################### +# Find or install Blosc2 +####################################### + +set(MINIMUM_BLOSC2_VERSION 2.11.0) +option(OPENEXR_FORCE_INTERNAL_BLOSC2 [=[Force using installed Blosc2.]=] OFF) + +set(BLOSC2_REPO "https://github.com/Blosc/c-blosc2.git" CACHE STRING "Repo path for blosc2 source") +set(BLOSC2_TAG "v${MINIMUM_BLOSC2_VERSION}" CACHE STRING "Tag to use for blosc2 source repo") + +if(NOT OPENEXR_FORCE_INTERNAL_BLOSC2) + #TODO: ^^ Release should not clone from main, this is a place holder + set(CMAKE_IGNORE_PATH "${CMAKE_CURRENT_BINARY_DIR}/_deps/blosc2-src/config;${CMAKE_CURRENT_BINARY_DIR}/_deps/blosc2-build/config") + find_package(Blosc2 ${MINIMUM_BLOSC2_VERSION}) + set(CMAKE_IGNORE_PATH) +endif() + +if(NOT TARGET Blosc2::blosc2_static AND NOT Blosc2_FOUND) + if(OPENEXR_FORCE_INTERNAL_BLOSC2) + message(STATUS "Blosc2 forced internal, installing from ${BLOSC2_REPO} (${BLOSC2_TAG})") + else() + message(STATUS "Blosc2 was not found, installing from ${BLOSC2_REPO} (${BLOSC2_TAG})") + endif() + + # configure the blosc2 build + set(BUILD_BENCHMARKS OFF CACHE INTERNAL "no benchmarks") + set(BUILD_EXAMPLES OFF CACHE INTERNAL "no examples") + set(BUILD_FUZZERS OFF CACHE INTERNAL "no fuzzer") + set(BUILD_SHARED OFF CACHE INTERNAL "no shared library") + set(BUILD_TESTS OFF CACHE INTERNAL "no tests") + + include(FetchContent) + FetchContent_Declare(Blosc2 + GIT_REPOSITORY "${BLOSC2_REPO}" + GIT_TAG "${BLOSC2_TAG}" + GIT_SHALLOW ON + GIT_PROGRESS ON) + + FetchContent_GetProperties(Blosc2) + if(NOT Blosc2_POPULATED) + FetchContent_Populate(Blosc2) + # Propagate OpenEXR's setting for pkg-config generation to Blosc2: + # If OpenEXR is generating it, the internal Blosc2 should, too. + # TODO: Do we really need that here ? + # set(BLOSC2_INSTALL_PKG_CONFIG ${OPENEXR_INSTALL_PKG_CONFIG}) + add_subdirectory(${blosc2_SOURCE_DIR} ${blosc2_BINARY_DIR}) + endif() + # the install creates this but if we're using the library locally we + # haven't installed the header files yet, so need to extract those + # and make a variable for header only usage + if(NOT TARGET Blosc2::Blosc2Config) + # TODO: Do we really need that here ? + # get_target_property(blosc2inc blosc2_static INTERFACE_INCLUDE_DIRECTORIES) + # get_target_property(blosc2confinc blosc2_shared INTERFACE_INCLUDE_DIRECTORIES) + # list(APPEND blosc2inc ${blosc2confinc}) + # set(BLOSC2_HEADER_ONLY_INCLUDE_DIRS ${blosc2inc}) + # message(STATUS "Blosc2 interface dirs ${BLOSC2_HEADER_ONLY_INCLUDE_DIRS}") + + get_target_property(blosc2inc Blosc2::blosc2_static INCLUDE_DIRECTORIES) + set(BLOSC2_INCLUDE_DIRS ${blosc2inc}) + + get_target_property(blosc2libdir Blosc2::blosc2_static BINARY_DIR) + set(BLOSC2_LIB_DIR ${blosc2libdir}) + + get_target_property(blosc2libname Blosc2::blosc2_static OUTPUT_NAME) + set(BLOSC2_LIB_NAME ${blosc2libname}) + endif() +else() + message(STATUS "Using Blosc2 ${Blosc2_VERSION} from ${Blosc2_DIR}") + # local build + if(NOT TARGET Blosc2::Blosc2Config AND TARGET Blosc2 AND TARGET Blosc2Config) + get_target_property(blosc2inc blosc2_static INTERFACE_INCLUDE_DIRECTORIES) + get_target_property(blosc2confinc blosc2_shared INTERFACE_INCLUDE_DIRECTORIES) + list(APPEND blosc2inc ${blosc2confinc}) + set(BLOSC2_HEADER_ONLY_INCLUDE_DIRS ${blosc2inc}) + message(STATUS "Blosc2 internal interface dirs: ${BLOSC2_HEADER_ONLY_INCLUDE_DIRS}") + + get_target_property(blosc2inc Blosc2::blosc2_static INCLUDE_DIRECTORIES) + set(BLOSC2_INCLUDE_DIRS ${blosc2inc}) + + get_target_property(blosc2libdir Blosc2::blosc2_static BINARY_DIR) + set(BLOSC2_LIB_DIR ${blosc2libdir}) + + get_target_property(blosc2libname Blosc2::blosc2_static OUTPUT_NAME) + set(BLOSC2_LIB_NAME ${blosc2libname}) + endif() +endif() + ########################################### # Check if we need to emulate vld1q_f32_x2 ########################################### diff --git a/src/lib/OpenEXR/CMakeLists.txt b/src/lib/OpenEXR/CMakeLists.txt index 45a809f386..1cccbdd8be 100644 --- a/src/lib/OpenEXR/CMakeLists.txt +++ b/src/lib/OpenEXR/CMakeLists.txt @@ -217,12 +217,9 @@ openexr_define_library(OpenEXR ImfXdr.h DEPENDENCIES Imath::Imath + Blosc2::blosc2_static OpenEXR::Config OpenEXR::Iex OpenEXR::IlmThread OpenEXR::OpenEXRCore ) - -target_include_directories(OpenEXR PUBLIC "/home/vladal/bin/include/") -target_link_directories(OpenEXR PUBLIC "/home/vladal/bin/lib") -target_link_libraries(OpenEXR PUBLIC "dl" "blosc2") \ No newline at end of file From 78ca6c60e325f48033038f74ce3e8229f424cd1e Mon Sep 17 00:00:00 2001 From: Vlad Lazar Date: Mon, 22 Jan 2024 13:50:48 -0500 Subject: [PATCH 05/34] Implement the compressor inside OpenEXRCore (#5) * Whitespaces and licensing * WIP OpenEXRCore implementation * Brand new spanking blosc build. * Switch to single Scanline zstd compression and Single implementation * Fixed the tests * Undo whitespace changes * Last touches * Revert extra build changes --- src/lib/OpenEXR/CMakeLists.txt | 1 - src/lib/OpenEXR/ImfCompressor.cpp | 7 +- src/lib/OpenEXR/ImfMultiPartInputFile.cpp | 3 +- src/lib/OpenEXR/ImfZstdCompressor.cpp | 129 ++++------------------ src/lib/OpenEXR/ImfZstdCompressor.h | 25 +---- src/lib/OpenEXRCore/CMakeLists.txt | 4 +- src/lib/OpenEXRCore/decoding.c | 4 + src/lib/OpenEXRCore/encoding.c | 1 + src/lib/OpenEXRCore/internal_compress.h | 1 + src/lib/OpenEXRCore/internal_decompress.h | 7 ++ src/lib/OpenEXRCore/internal_zstd.c | 115 +++++++++++++++++++ src/lib/OpenEXRCore/openexr_attr.h | 1 + src/lib/OpenEXRCore/openexr_compression.h | 11 ++ src/lib/OpenEXRCore/parse_header.c | 2 + src/test/OpenEXRCoreTest/compression.cpp | 12 ++ src/test/OpenEXRCoreTest/compression.h | 2 + src/test/OpenEXRCoreTest/main.cpp | 2 + 17 files changed, 194 insertions(+), 133 deletions(-) create mode 100644 src/lib/OpenEXRCore/internal_zstd.c diff --git a/src/lib/OpenEXR/CMakeLists.txt b/src/lib/OpenEXR/CMakeLists.txt index 1cccbdd8be..c6759e7031 100644 --- a/src/lib/OpenEXR/CMakeLists.txt +++ b/src/lib/OpenEXR/CMakeLists.txt @@ -217,7 +217,6 @@ openexr_define_library(OpenEXR ImfXdr.h DEPENDENCIES Imath::Imath - Blosc2::blosc2_static OpenEXR::Config OpenEXR::Iex OpenEXR::IlmThread diff --git a/src/lib/OpenEXR/ImfCompressor.cpp b/src/lib/OpenEXR/ImfCompressor.cpp index dfb5ab8adf..db51021dc6 100644 --- a/src/lib/OpenEXR/ImfCompressor.cpp +++ b/src/lib/OpenEXR/ImfCompressor.cpp @@ -19,6 +19,7 @@ #include "ImfRleCompressor.h" #include "ImfZipCompressor.h" #include "ImfZstdCompressor.h" +#include "openexr_compression.h" OPENEXR_IMF_INTERNAL_NAMESPACE_SOURCE_ENTER @@ -145,7 +146,7 @@ newCompressor (Compression c, size_t maxScanLineSize, const Header& hdr) DwaCompressor::STATIC_HUFFMAN); case ZSTD_COMPRESSION: - return new ZstdCompressor (hdr, maxScanLineSize, 32); + return new ZstdCompressor (hdr); default: return 0; } } @@ -167,7 +168,7 @@ numLinesInBuffer (Compression comp) case B44_COMPRESSION: case B44A_COMPRESSION: case DWAA_COMPRESSION: return 32; - case ZSTD_COMPRESSION: return 32; + case ZSTD_COMPRESSION: return (int)exr_get_zstd_lines_per_chunk(); case DWAB_COMPRESSION: return 256; default: throw IEX_NAMESPACE::ArgExc ("Unknown compression type"); @@ -190,7 +191,7 @@ newTileCompressor ( return new ZipCompressor (hdr, tileLineSize, numTileLines); case ZSTD_COMPRESSION: - return new ZstdCompressor (hdr, tileLineSize, numTileLines); + return new ZstdCompressor (hdr); case PIZ_COMPRESSION: diff --git a/src/lib/OpenEXR/ImfMultiPartInputFile.cpp b/src/lib/OpenEXR/ImfMultiPartInputFile.cpp index b9ed6d5d29..b6fb9aeb12 100644 --- a/src/lib/OpenEXR/ImfMultiPartInputFile.cpp +++ b/src/lib/OpenEXR/ImfMultiPartInputFile.cpp @@ -22,6 +22,7 @@ #include "ImfTiledMisc.h" #include "ImfTimeCodeAttribute.h" #include "ImfVersion.h" +#include "openexr_compression.h" #include @@ -547,7 +548,7 @@ MultiPartInputFile::Data::chunkOffsetReconstruction ( // (TODO) fix this so that it doesn't need to be revised for future compression types. switch (parts[i]->header.compression ()) { - case ZSTD_COMPRESSION: rowsizes[i] = 32; break; + case ZSTD_COMPRESSION: rowsizes[i] = (int)exr_get_zstd_lines_per_chunk(); break; case DWAB_COMPRESSION: rowsizes[i] = 256; break; case PIZ_COMPRESSION: case B44_COMPRESSION: diff --git a/src/lib/OpenEXR/ImfZstdCompressor.cpp b/src/lib/OpenEXR/ImfZstdCompressor.cpp index d35a3deecd..55b4a8cea1 100644 --- a/src/lib/OpenEXR/ImfZstdCompressor.cpp +++ b/src/lib/OpenEXR/ImfZstdCompressor.cpp @@ -4,75 +4,42 @@ // #include +#include +#include "openexr_compression.h" #include "ImfZstdCompressor.h" -#include "blosc2.h" #include "IlmThreadPool.h" #include "ImfChannelList.h" #include "ImfMisc.h" + namespace { -class BloscInit -{ -public: - static void Init () { getInstance (); } - BloscInit (const BloscInit&) = delete; - BloscInit& operator= (const BloscInit&) = delete; - -private: - BloscInit () { blosc2_init (); } - ~BloscInit () { blosc2_destroy (); } - static BloscInit& getInstance () - { - static BloscInit instance; - return instance; - } -}; -} // namespace - +std::mutex g_mutex; +} + OPENEXR_IMF_INTERNAL_NAMESPACE_SOURCE_ENTER -ZstdCompressor::ZstdCompressor ( - const Header& hdr, size_t maxScanlineSize, size_t numScanLines) - : Compressor (hdr) - , _maxScanlineSize (maxScanlineSize) - , _numScanLines (numScanLines) - , _outBuffer (nullptr, &free) - , _schunk (nullptr, &blosc2_schunk_free) + +ZstdCompressor::ZstdCompressor (const Header& hdr) + : Compressor (hdr), _outBuffer () {} int ZstdCompressor::numScanLines () const { - return _numScanLines; // Needs to be in sync with ImfCompressor::numLinesInBuffer + return (int)exr_get_zstd_lines_per_chunk(); // Needs to be in sync with ImfCompressor::numLinesInBuffer } int ZstdCompressor::compress ( const char* inPtr, int inSize, int minY, const char*& outPtr) { - int typeSize = std::numeric_limits::min (); - for (auto it = header ().channels ().begin (); - it != header ().channels ().end (); - ++it) + outPtr = (char*) malloc (inSize); { - // BLOSC prefilter is affected by the typesize. Initializing to max will ensure that a channel is not split in 2 and filtered separately. - // probably compression can be improved for non-deep images by compressing every channel separately with the correct typeSize - // (much harder to do for Deep-Data). - typeSize = std::max (typeSize, Imf::pixelTypeSize (it.channel ().type)); + std::lock_guard lock (g_mutex); + _outBuffer.push_back (raw_ptr ((char*) outPtr, &free)); } - - auto ret = BLOSC_compress_impl (inPtr, inSize, typeSize, outPtr); - auto fullSize = Xdr::size () + ret; - auto data = malloc (fullSize); - auto write = (char*) data; - - Xdr::write (write, Versions::LATEST); - - memcpy (write, outPtr, ret); - outPtr = (char*) data; - - _outBuffer = raw_ptr ((char*) data, &free); - + auto fullSize = + exr_compress_zstd ((char*) (inPtr), inSize, (void*) outPtr, inSize); return fullSize; } @@ -80,67 +47,15 @@ int ZstdCompressor::uncompress ( const char* inPtr, int inSize, int minY, const char*& outPtr) { - auto read = (const char*) inPtr; - int v; - Xdr::read (read, v); - if (v == Versions::SINGLE_BLOB) + auto read = (const char*) inPtr; + void* write = nullptr; + auto ret = exr_uncompress_zstd (read, inSize, &write, 0); { - return BLOSC_uncompress_impl_single_blob ( - read, inSize - Xdr::size (), outPtr); + std::lock_guard lock (g_mutex); + _outBuffer.push_back (raw_ptr ((char*) write, &free)); } - else { throw Iex::InputExc ("Unsupported ZstdCompressor version"); } -} - -int -ZstdCompressor::BLOSC_compress_impl ( - const char* inPtr, int inSize, int typeSize, const char*& out) -{ - BloscInit::Init (); - blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS; - - cparams.typesize = typeSize; - // clevel 9 is about a 20% increase in compression compared to 5. - // Decompression speed is unchanged. - cparams.clevel = header ().zstdCompressionLevel (); - cparams.nthreads = 1; - cparams.compcode = BLOSC_ZSTD; // Codec - cparams.splitmode = - BLOSC_NEVER_SPLIT; // Split => multithreading, not split better compression - - blosc2_storage storage = BLOSC2_STORAGE_DEFAULTS; - storage.cparams = &cparams; - storage.contiguous = true; - - _schunk = schunk_ptr (blosc2_schunk_new (&storage), &blosc2_schunk_free); - - auto in = const_cast (inPtr); - blosc2_schunk_append_buffer (_schunk.get (), in, inSize); - - uint8_t* buffer; - bool shouldFree = true; - auto size = blosc2_schunk_to_buffer (_schunk.get (), &buffer, &shouldFree); - out = (char*) buffer; - if (shouldFree) { _outBuffer = raw_ptr ((char*) buffer, &free); } - return size; -} - -int -ZstdCompressor::BLOSC_uncompress_impl_single_blob ( - const char* inPtr, int inSize, const char*& out) -{ - auto in = const_cast (inPtr); - _schunk = schunk_ptr ( - blosc2_schunk_from_buffer ( - reinterpret_cast (in), inSize, true), - &blosc2_schunk_free); - - auto buffSize = _maxScanlineSize * numScanLines (); - _outBuffer = - Imf::ZstdCompressor::raw_ptr ((char*) malloc (buffSize), &free); - auto size = blosc2_schunk_decompress_chunk ( - _schunk.get (), 0, _outBuffer.get (), buffSize); - out = _outBuffer.get (); - return size; + outPtr = (const char*) write; + return ret; } OPENEXR_IMF_INTERNAL_NAMESPACE_SOURCE_EXIT \ No newline at end of file diff --git a/src/lib/OpenEXR/ImfZstdCompressor.h b/src/lib/OpenEXR/ImfZstdCompressor.h index d9319e076c..c9f045cddd 100644 --- a/src/lib/OpenEXR/ImfZstdCompressor.h +++ b/src/lib/OpenEXR/ImfZstdCompressor.h @@ -1,45 +1,30 @@ -#pragma once - // // SPDX-License-Identifier: BSD-3-Clause // Copyright (c) Contributors to the OpenEXR Project. // +#pragma once + #include #include "ImfNamespace.h" #include "ImfCompressor.h" #include "ImfHeader.h" #include "blosc2.h" +#include "vector" OPENEXR_IMF_INTERNAL_NAMESPACE_HEADER_ENTER class ZstdCompressor : public Compressor { public: - ZstdCompressor ( - const Header& hdr, size_t maxScanLines, size_t numScanLines); - + explicit ZstdCompressor (const Header& hdr); private: - using schunk_ptr = - std::unique_ptr; using raw_ptr = std::unique_ptr; - raw_ptr _outBuffer; - schunk_ptr _schunk; - size_t _maxScanlineSize; - size_t _numScanLines; + std::vector _outBuffer; int numScanLines () const override; // max int compress ( const char* inPtr, int inSize, int minY, const char*& outPtr) override; int uncompress ( const char* inPtr, int inSize, int minY, const char*& outPtr) override; - int BLOSC_compress_impl ( - const char* inPtr, int inSize, int typeSize, const char*& out); - int BLOSC_uncompress_impl_single_blob ( - const char* inPtr, int inSize, const char*& out); - enum Versions : int - { - SINGLE_BLOB = 1, - LATEST = SINGLE_BLOB - }; }; OPENEXR_IMF_INTERNAL_NAMESPACE_HEADER_EXIT \ No newline at end of file diff --git a/src/lib/OpenEXRCore/CMakeLists.txt b/src/lib/OpenEXRCore/CMakeLists.txt index f4bd54efb3..ecfc171202 100644 --- a/src/lib/OpenEXRCore/CMakeLists.txt +++ b/src/lib/OpenEXRCore/CMakeLists.txt @@ -45,6 +45,7 @@ openexr_define_library(OpenEXRCore internal_piz.c internal_dwa.c internal_huf.c + internal_zstd.c attributes.c string.c @@ -102,6 +103,7 @@ openexr_define_library(OpenEXRCore DEPENDENCIES Imath::Imath + Blosc2::blosc2_static ) if (DEFINED EXR_DEFLATE_LIB) @@ -110,4 +112,4 @@ if (DEFINED EXR_DEFLATE_LIB) else() target_link_libraries(OpenEXRCore PUBLIC ${EXR_DEFLATE_LIB}) endif() -endif() +endif() \ No newline at end of file diff --git a/src/lib/OpenEXRCore/decoding.c b/src/lib/OpenEXRCore/decoding.c index 322cbd8965..c7db79f755 100644 --- a/src/lib/OpenEXRCore/decoding.c +++ b/src/lib/OpenEXRCore/decoding.c @@ -263,6 +263,10 @@ decompress_data ( rv = internal_exr_undo_dwab ( decode, packbufptr, packsz, unpackbufptr, unpacksz); break; + case EXR_COMPRESSION_ZSTD: + rv = internal_exr_undo_zstd ( + decode, packbufptr, packsz, unpackbufptr, unpacksz); + break; case EXR_COMPRESSION_LAST_TYPE: default: return pctxt->print_error ( diff --git a/src/lib/OpenEXRCore/encoding.c b/src/lib/OpenEXRCore/encoding.c index b92e0ce356..6b017ed7b0 100644 --- a/src/lib/OpenEXRCore/encoding.c +++ b/src/lib/OpenEXRCore/encoding.c @@ -54,6 +54,7 @@ default_compress_chunk (exr_encode_pipeline_t* encode) case EXR_COMPRESSION_B44A: rv = internal_exr_apply_b44a (encode); break; case EXR_COMPRESSION_DWAA: rv = internal_exr_apply_dwaa (encode); break; case EXR_COMPRESSION_DWAB: rv = internal_exr_apply_dwab (encode); break; + case EXR_COMPRESSION_ZSTD: rv = internal_exr_apply_zstd (encode); break; case EXR_COMPRESSION_LAST_TYPE: default: return pctxt->print_error ( diff --git a/src/lib/OpenEXRCore/internal_compress.h b/src/lib/OpenEXRCore/internal_compress.h index 360015c120..27315f68a0 100644 --- a/src/lib/OpenEXRCore/internal_compress.h +++ b/src/lib/OpenEXRCore/internal_compress.h @@ -33,4 +33,5 @@ exr_result_t internal_exr_apply_dwaa (exr_encode_pipeline_t* encode); exr_result_t internal_exr_apply_dwab (exr_encode_pipeline_t* encode); +exr_result_t internal_exr_apply_zstd (exr_encode_pipeline_t* encode); #endif /* OPENEXR_CORE_COMPRESS_H */ diff --git a/src/lib/OpenEXRCore/internal_decompress.h b/src/lib/OpenEXRCore/internal_decompress.h index 834b854a3e..2726a4afd3 100644 --- a/src/lib/OpenEXRCore/internal_decompress.h +++ b/src/lib/OpenEXRCore/internal_decompress.h @@ -73,4 +73,11 @@ exr_result_t internal_exr_undo_dwab ( void* uncompressed_data, uint64_t uncompressed_size); +exr_result_t internal_exr_undo_zstd ( + exr_decode_pipeline_t* decode, + const void* compressed_data, + uint64_t comp_buf_size, + void* uncompressed_data, + uint64_t uncompressed_size); + #endif /* OPENEXR_CORE_DECOMPRESS_H */ diff --git a/src/lib/OpenEXRCore/internal_zstd.c b/src/lib/OpenEXRCore/internal_zstd.c new file mode 100644 index 0000000000..1fc845c6a0 --- /dev/null +++ b/src/lib/OpenEXRCore/internal_zstd.c @@ -0,0 +1,115 @@ +/* +** SPDX-License-Identifier: BSD-3-Clause +** Copyright Contributors to the OpenEXR Project. +*/ + +#include +#include "internal_compress.h" +#include "internal_decompress.h" +#include "blosc2.h" + +size_t +exr_get_zstd_lines_per_chunk () +{ + return 1; +} + +long +exr_compress_zstd (char* inPtr, int inSize, void* outPtr, int outPtrSize) +{ + if (inSize == 0) // Weird input data when subsampling + { + outPtr = NULL; + return 0; + } + + blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS; + int typeSize = inSize % 4 == 0 ? 4 : 2; + cparams.typesize = typeSize; + // clevel 9 is about a 20% increase in compression compared to 5. + // Decompression speed is unchanged. + int zstd_level; + exr_get_default_zstd_compression_level (&zstd_level); + cparams.clevel = zstd_level; + cparams.nthreads = 1; + cparams.compcode = BLOSC_ZSTD; // Codec + cparams.splitmode = + BLOSC_NEVER_SPLIT; // Split => multithreading, not split better compression + + blosc2_storage storage = BLOSC2_STORAGE_DEFAULTS; + storage.contiguous = true; + storage.cparams = &cparams; + + blosc2_schunk* _schunk = blosc2_schunk_new (&storage); + + blosc2_schunk_append_buffer (_schunk, inPtr, inSize); + + uint8_t* buffer; + bool shouldFree = true; + int64_t size = blosc2_schunk_to_buffer (_schunk, &buffer, &shouldFree); + + if (size <= inSize && size <= outPtrSize && size > 0) + { memcpy (outPtr, buffer, size); } + else + { + memcpy (outPtr, inPtr, inSize); + size = inSize; // We increased compression size + } + + if (shouldFree) { free (buffer); } + + blosc2_schunk_free (_schunk); + return size; +} + +long +exr_uncompress_zstd ( + const char* inPtr, uint64_t inSize, void** outPtr, uint64_t outPtrSize) +{ + blosc2_schunk* _schunk = blosc2_schunk_from_buffer ((uint8_t *)inPtr, inSize, true); + + if (_schunk == NULL) { return -1; } + + if (outPtrSize == 0) // we don't have any storage allocated + { + *outPtr = malloc (_schunk->nbytes); + outPtrSize = _schunk->nbytes; + } + + int size = blosc2_schunk_decompress_chunk (_schunk, 0, *outPtr, outPtrSize); + blosc2_schunk_free (_schunk); + + return size; +} + +exr_result_t +internal_exr_apply_zstd (exr_encode_pipeline_t* encode) +{ + long compressedSize = exr_compress_zstd ( + encode->packed_buffer, + encode->packed_bytes, + encode->compressed_buffer, + encode->compressed_alloc_size); + if (compressedSize < 0) { return EXR_ERR_UNKNOWN; } + + encode->compressed_bytes = compressedSize; + return EXR_ERR_SUCCESS; +} + +exr_result_t +internal_exr_undo_zstd ( + exr_decode_pipeline_t* decode, + const void* compressed_data, + uint64_t comp_buf_size, + void* uncompressed_data, + uint64_t uncompressed_size) +{ + + long uncompressedSize = exr_uncompress_zstd ( + (const char*) compressed_data, + comp_buf_size, + &uncompressed_data, + uncompressed_size); + if (uncompressed_size != uncompressedSize) { return EXR_ERR_CORRUPT_CHUNK; } + return EXR_ERR_SUCCESS; +} \ No newline at end of file diff --git a/src/lib/OpenEXRCore/openexr_attr.h b/src/lib/OpenEXRCore/openexr_attr.h index eabcd57d9c..e6aa7421ba 100644 --- a/src/lib/OpenEXRCore/openexr_attr.h +++ b/src/lib/OpenEXRCore/openexr_attr.h @@ -45,6 +45,7 @@ typedef enum EXR_COMPRESSION_B44A = 7, EXR_COMPRESSION_DWAA = 8, EXR_COMPRESSION_DWAB = 9, + EXR_COMPRESSION_ZSTD = 10, EXR_COMPRESSION_LAST_TYPE /**< Invalid value, provided for range checking. */ } exr_compression_t; diff --git a/src/lib/OpenEXRCore/openexr_compression.h b/src/lib/OpenEXRCore/openexr_compression.h index 67ae45004b..e5956cfc85 100644 --- a/src/lib/OpenEXRCore/openexr_compression.h +++ b/src/lib/OpenEXRCore/openexr_compression.h @@ -45,6 +45,17 @@ exr_result_t exr_uncompress_buffer ( size_t out_bytes_avail, size_t* actual_out); +EXR_EXPORT +long exr_compress_zstd ( + char* inPtr, int inSize, void * outPtr, int outPtrSize); + +EXR_EXPORT +long exr_uncompress_zstd ( + const char* inPtr, uint64_t inSize, void ** outPtr, uint64_t outPtrSize); + +EXR_EXPORT +size_t exr_get_zstd_lines_per_chunk(); + #ifdef __cplusplus } /* extern "C" */ #endif diff --git a/src/lib/OpenEXRCore/parse_header.c b/src/lib/OpenEXRCore/parse_header.c index b85273a378..a7acc4621d 100644 --- a/src/lib/OpenEXRCore/parse_header.c +++ b/src/lib/OpenEXRCore/parse_header.c @@ -9,6 +9,7 @@ #include "internal_constants.h" #include "internal_structs.h" #include "internal_xdr.h" +#include "openexr_compression.h" #include #include @@ -2364,6 +2365,7 @@ internal_exr_compute_chunk_offset_size (struct _internal_exr_part* curpart) case EXR_COMPRESSION_B44A: case EXR_COMPRESSION_DWAA: linePerChunk = 32; break; case EXR_COMPRESSION_DWAB: linePerChunk = 256; break; + case EXR_COMPRESSION_ZSTD: linePerChunk = exr_get_zstd_lines_per_chunk(); break; case EXR_COMPRESSION_LAST_TYPE: default: /* ERROR CONDITION */ diff --git a/src/test/OpenEXRCoreTest/compression.cpp b/src/test/OpenEXRCoreTest/compression.cpp index 772111647f..e0764e9f0b 100644 --- a/src/test/OpenEXRCoreTest/compression.cpp +++ b/src/test/OpenEXRCoreTest/compression.cpp @@ -1419,6 +1419,7 @@ doWriteRead ( case EXR_COMPRESSION_RLE: case EXR_COMPRESSION_ZIP: case EXR_COMPRESSION_ZIPS: + case EXR_COMPRESSION_ZSTD: restore.compareExact (p, "orig", "C loaded C"); break; case EXR_COMPRESSION_PIZ: @@ -1681,6 +1682,12 @@ testDWABCompression (const std::string& tempdir) testComp (tempdir, EXR_COMPRESSION_DWAB); } +void +testZstdCompression (const std::string& tempdir) +{ + testComp (tempdir, EXR_COMPRESSION_ZSTD); +} + void testDeepNoCompression (const std::string& tempdir) {} @@ -1692,3 +1699,8 @@ testDeepZIPCompression (const std::string& tempdir) void testDeepZIPSCompression (const std::string& tempdir) {} + +void +testDeepZstdCompression (const std::string& tempdir) +{ +} \ No newline at end of file diff --git a/src/test/OpenEXRCoreTest/compression.h b/src/test/OpenEXRCoreTest/compression.h index 573e10f96c..ef5391a92b 100644 --- a/src/test/OpenEXRCoreTest/compression.h +++ b/src/test/OpenEXRCoreTest/compression.h @@ -18,9 +18,11 @@ void testB44Compression (const std::string& tempdir); void testB44ACompression (const std::string& tempdir); void testDWAACompression (const std::string& tempdir); void testDWABCompression (const std::string& tempdir); +void testZstdCompression (const std::string& tempdir); void testDeepNoCompression (const std::string& tempdir); void testDeepZIPCompression (const std::string& tempdir); void testDeepZIPSCompression (const std::string& tempdir); +void testDeepZstdCompression (const std::string& tempdir); #endif // OPENEXR_CORE_TEST_COMPRESSION_H diff --git a/src/test/OpenEXRCoreTest/main.cpp b/src/test/OpenEXRCoreTest/main.cpp index d12d6718f7..dae1ea29ae 100644 --- a/src/test/OpenEXRCoreTest/main.cpp +++ b/src/test/OpenEXRCoreTest/main.cpp @@ -202,10 +202,12 @@ main (int argc, char* argv[]) TEST (testB44ACompression, "core_compression"); TEST (testDWAACompression, "core_compression"); TEST (testDWABCompression, "core_compression"); + TEST (testZstdCompression, "core_compression"); TEST (testDeepNoCompression, "core_compression"); TEST (testDeepZIPCompression, "core_compression"); TEST (testDeepZIPSCompression, "core_compression"); + TEST (testDeepZstdCompression, "core_compression"); // empty dummy test if (helpMode) { From 41afe2e36e7aa4e366c466c8899f08bb0deb0a90 Mon Sep 17 00:00:00 2001 From: Vlad-Andrei Lazar Date: Mon, 8 Jan 2024 15:24:10 -0500 Subject: [PATCH 06/34] Fix single blob Signed-off-by: Vlad-Andrei Lazar --- src/examples/deepTiledExamples.cpp | 147 ++++++++++++++------------ src/lib/OpenEXR/ImfZstdCompressor.cpp | 14 ++- 2 files changed, 83 insertions(+), 78 deletions(-) diff --git a/src/examples/deepTiledExamples.cpp b/src/examples/deepTiledExamples.cpp index 3e58600c09..4f2d22f3c2 100644 --- a/src/examples/deepTiledExamples.cpp +++ b/src/examples/deepTiledExamples.cpp @@ -28,17 +28,12 @@ using namespace IMF; using namespace std; using namespace IMATH_NAMESPACE; - // defined in deepExamples.cpp extern Array2D testDataZ; -extern Array2D testDataA; -extern unsigned int getPixelSampleCount (int i, int j); -extern void getPixelSampleData( - int i, - int j, - Array2D& dataZ, - Array2D& dataA); - +extern Array2D testDataA; +extern unsigned int getPixelSampleCount (int i, int j); +extern void getPixelSampleData ( + int i, int j, Array2D& dataZ, Array2D& dataA); void readDeepTiledFile ( @@ -62,24 +57,24 @@ readDeepTiledFile ( // - allocate the memory requred to store the samples // - read the pixels from the file // - + DeepTiledInputFile file (filename); - + int width = dataWindow.max.x - dataWindow.min.x + 1; int height = dataWindow.max.y - dataWindow.min.y + 1; - + sampleCount.resizeErase (height, width); dataZ.resizeErase (height, width); dataA.resizeErase (height, width); - + DeepFrameBuffer frameBuffer; - + frameBuffer.insertSampleCountSlice (Slice ( UINT, (char*) (&sampleCount[0][0] - dataWindow.min.x - dataWindow.min.y * width), sizeof (unsigned int) * 1, // xStride sizeof (unsigned int) * width)); // yStride - + frameBuffer.insert ( "Z", DeepSlice ( @@ -88,7 +83,7 @@ readDeepTiledFile ( sizeof (float*) * 1, // xStride for pointer array sizeof (float*) * width, // yStride for pointer array sizeof (float) * 1)); // stride for samples - + frameBuffer.insert ( "A", DeepSlice ( @@ -97,14 +92,14 @@ readDeepTiledFile ( sizeof (half*) * 1, // xStride for pointer array sizeof (half*) * width, // yStride for pointer array sizeof (half) * 1)); // stride for samples - + file.setFrameBuffer (frameBuffer); - + int numXTiles = file.numXTiles (0); int numYTiles = file.numYTiles (0); - + file.readPixelSampleCounts (0, numXTiles - 1, 0, numYTiles - 1); - + for (int i = 0; i < height; i++) { for (int j = 0; j < width; j++) @@ -113,11 +108,11 @@ readDeepTiledFile ( dataA[i][j] = new half[sampleCount[i][j]]; } } - + file.readTiles (0, numXTiles - 1, 0, numYTiles - 1); - + // (after read data is processed, data must be freed:) - + for (int i = 0; i < height; i++) { for (int j = 0; j < width; j++) @@ -128,42 +123,43 @@ readDeepTiledFile ( } } -void getSampleDataForTile ( - int i, - int j, - int tileSizeX, - int tileSizeY, - Array2D& sampleCount, - Array2D& dataZ, - Array2D& dataA) +void +getSampleDataForTile ( + int i, + int j, + int tileSizeX, + int tileSizeY, + Array2D& sampleCount, + Array2D& dataZ, + Array2D& dataA) { for (int k = 0; k < tileSizeY; k++) { int y = j * tileSizeY + k; - if (y >= sampleCount.height()) break; - + if (y >= sampleCount.height ()) break; + for (int l = 0; l < tileSizeX; l++) { int x = i * tileSizeX + l; - if (x >= sampleCount.width()) break; - - sampleCount[y][x] = getPixelSampleCount(y, x); - + if (x >= sampleCount.width ()) break; + + sampleCount[y][x] = getPixelSampleCount (y, x); + dataZ[y][x] = new float[sampleCount[y][x]]; - dataA[y][x] = new half [sampleCount[y][x]]; - - getPixelSampleData(y, x, dataZ, dataA); + dataA[y][x] = new half[sampleCount[y][x]]; + + getPixelSampleData (y, x, dataZ, dataA); } } } void writeDeepTiledFile ( - const char filename[], - Box2i displayWindow, - Box2i dataWindow, - int tileSizeX, - int tileSizeY, + const char filename[], + Box2i displayWindow, + Box2i dataWindow, + int tileSizeX, + int tileSizeY, Compression compression = Compression::ZIPS_COMPRESSION) { // @@ -176,7 +172,7 @@ writeDeepTiledFile ( // - describe the memory layout of the A and Z pixels // - store the pixels in the file // - + int height = dataWindow.max.y - dataWindow.min.y + 1; int width = dataWindow.max.x - dataWindow.min.x + 1; @@ -233,11 +229,12 @@ writeDeepTiledFile ( for (int i = 0; i < file.numXTiles (0); i++) { // Generate data for sampleCount, dataZ and dataA. - getSampleDataForTile (i, j, tileSizeX, tileSizeY, sampleCount, dataZ, dataA); + getSampleDataForTile ( + i, j, tileSizeX, tileSizeY, sampleCount, dataZ, dataA); file.writeTile (i, j, 0); } } - + for (int i = 0; i < height; i++) { for (int j = 0; j < width; j++) @@ -248,34 +245,44 @@ writeDeepTiledFile ( } } -void deepTiledExamples() +void +deepTiledExamples () { int w = 800; int h = 600; - + int tileSizeX = 64; int tileSizeY = 64; - + Box2i window; - window.min.setValue(0, 0); - window.max.setValue(w - 1, h - 1); - - Array2D dataZ; - dataZ.resizeErase(h, w); - - Array2D dataA; - dataA.resizeErase(h, w); - + window.min.setValue (0, 0); + window.max.setValue (w - 1, h - 1); + + Array2D dataZ; + dataZ.resizeErase (h, w); + + Array2D dataA; + dataA.resizeErase (h, w); + Array2D sampleCount; - sampleCount.resizeErase(h, w); - + sampleCount.resizeErase (h, w); + // Create an image to be used as a source for deep data - testDataA.resizeErase(h, w); - testDataZ.resizeErase(h, w); - drawImage2(testDataA, testDataZ, w, h); - - writeDeepTiledFile("testTiled.deep.zip.exr", window, window, tileSizeX, tileSizeY); - readDeepTiledFile ("testTiled.deep.zip.exr", window, window, dataZ, dataA, sampleCount); - writeDeepTiledFile("testTiled.deep.zstd.exr", window, window, tileSizeX, tileSizeY, Compression::ZSTD_COMPRESSION); - readDeepTiledFile ("testTiled.deep.zstd.exr", window, window, dataZ, dataA, sampleCount); + testDataA.resizeErase (h, w); + testDataZ.resizeErase (h, w); + drawImage2 (testDataA, testDataZ, w, h); + + writeDeepTiledFile ( + "testTiled.deep.exr", window, window, tileSizeX, tileSizeY); + readDeepTiledFile ( + "testTiled.deep.exr", window, window, dataZ, dataA, sampleCount); + writeDeepTiledFile ( + "testTiled.deep.zstd.exr", + window, + window, + tileSizeX, + tileSizeY, + Compression::ZSTD_COMPRESSION); + readDeepTiledFile ( + "testTiled.deep.zstd.exr", window, window, dataZ, dataA, sampleCount); } diff --git a/src/lib/OpenEXR/ImfZstdCompressor.cpp b/src/lib/OpenEXR/ImfZstdCompressor.cpp index 43fa4c640c..48e742d591 100644 --- a/src/lib/OpenEXR/ImfZstdCompressor.cpp +++ b/src/lib/OpenEXR/ImfZstdCompressor.cpp @@ -56,9 +56,10 @@ ZstdCompressor::compress ( typeSize = std::max (typeSize, Imf::pixelTypeSize (it.channel ().type)); } - auto ret = BLOSC_compress_impl (inPtr, inSize, typeSize, outPtr); - auto data = malloc (Xdr::size () + ret); - auto write = (char*) data; + auto ret = BLOSC_compress_impl (inPtr, inSize, typeSize, outPtr); + auto fullSize = Xdr::size () + ret; + auto data = malloc (fullSize); + auto write = (char*) data; Xdr::write (write, Versions::LATEST); @@ -67,7 +68,7 @@ ZstdCompressor::compress ( _outBuffer = raw_ptr ((char*) data, &free); - return ret; + return fullSize; } int @@ -82,10 +83,7 @@ ZstdCompressor::uncompress ( return BLOSC_uncompress_impl_single_blob ( read, inSize - Xdr::size (), outPtr); } - else - { - throw Iex::InputExc ("Unsupported ZstdCompressor version"); - } + else { throw Iex::InputExc ("Unsupported ZstdCompressor version"); } } int From f52266d1ebd06c8f0fe1019ad1b8624c0bfc5d39 Mon Sep 17 00:00:00 2001 From: Philippe Leprince Date: Wed, 10 Jan 2024 17:19:28 +0100 Subject: [PATCH 07/34] Add license headers. Clang-format example file. Clang-format deepExamples.cpp and fix a comment typo. Clang-format deepExamples.cpp and fix a comment typo. Signed-off-by: Vlad-Andrei Lazar --- src/examples/deepExamples.cpp | 20 +++++++++++++------- src/lib/OpenEXR/ImfZstdCompressor.cpp | 5 +++++ src/lib/OpenEXR/ImfZstdCompressor.h | 6 ++++++ 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/src/examples/deepExamples.cpp b/src/examples/deepExamples.cpp index b28a0cf3bd..f7599084d5 100644 --- a/src/examples/deepExamples.cpp +++ b/src/examples/deepExamples.cpp @@ -47,7 +47,7 @@ readDeepScanlineFile ( // - allocate memory for the pixels // - describe the layout of the A, and Z pixel buffers // - read the sample counts from the file - // - allocate the memory requred to store the samples + // - allocate the memory required to store the samples // - read the pixels from the file // @@ -69,7 +69,8 @@ readDeepScanlineFile ( frameBuffer.insertSampleCountSlice (Slice ( UINT, - (char*) (&sampleCount[0][0] - dataWindow.min.x - dataWindow.min.y * width), + (char*) (&sampleCount[0][0] - dataWindow.min.x - + dataWindow.min.y * width), sizeof (unsigned int) * 1, // xStride sizeof (unsigned int) * width)); // yStride @@ -77,7 +78,8 @@ readDeepScanlineFile ( "dataZ", DeepSlice ( FLOAT, - (char*) (&dataZ[0][0] - dataWindow.min.x - dataWindow.min.y * width), + (char*) (&dataZ[0][0] - dataWindow.min.x - + dataWindow.min.y * width), sizeof (float*) * 1, // xStride for pointer array sizeof (float*) * width, // yStride for pointer array @@ -87,7 +89,8 @@ readDeepScanlineFile ( "dataA", DeepSlice ( HALF, - (char*) (&dataA[0][0] - dataWindow.min.x - dataWindow.min.y * width), + (char*) (&dataA[0][0] - dataWindow.min.x - + dataWindow.min.y * width), sizeof (half*) * 1, // xStride for pointer array sizeof (half*) * width, // yStride for pointer array sizeof (half) * 1)); // stride for O data sample @@ -176,7 +179,8 @@ writeDeepScanlineFile ( frameBuffer.insertSampleCountSlice (Slice ( UINT, - (char*) (&sampleCount[0][0] - dataWindow.min.x - dataWindow.min.y * width), + (char*) (&sampleCount[0][0] - dataWindow.min.x - + dataWindow.min.y * width), sizeof (unsigned int) * 1, // xS sizeof (unsigned int) * width)); // yStride @@ -185,7 +189,8 @@ writeDeepScanlineFile ( "Z", DeepSlice ( FLOAT, - (char*) (&dataZ[0][0] - dataWindow.min.x - dataWindow.min.y * width), + (char*) (&dataZ[0][0] - dataWindow.min.x - + dataWindow.min.y * width), sizeof (float*) * 1, // xStride for pointer sizeof (float*) * width, // yStride for pointer array @@ -195,7 +200,8 @@ writeDeepScanlineFile ( "A", DeepSlice ( HALF, - (char*) (&dataA[0][0] - dataWindow.min.x - dataWindow.min.y * width), + (char*) (&dataA[0][0] - dataWindow.min.x - + dataWindow.min.y * width), sizeof (half*) * 1, // xStride for pointer array sizeof (half*) * width, // yStride for pointer array sizeof (half) * 1)); // stride for A data sample diff --git a/src/lib/OpenEXR/ImfZstdCompressor.cpp b/src/lib/OpenEXR/ImfZstdCompressor.cpp index 48e742d591..d35a3deecd 100644 --- a/src/lib/OpenEXR/ImfZstdCompressor.cpp +++ b/src/lib/OpenEXR/ImfZstdCompressor.cpp @@ -1,3 +1,8 @@ +// +// SPDX-License-Identifier: BSD-3-Clause +// Copyright (c) Contributors to the OpenEXR Project. +// + #include #include "ImfZstdCompressor.h" diff --git a/src/lib/OpenEXR/ImfZstdCompressor.h b/src/lib/OpenEXR/ImfZstdCompressor.h index f1c9110dd0..d9319e076c 100644 --- a/src/lib/OpenEXR/ImfZstdCompressor.h +++ b/src/lib/OpenEXR/ImfZstdCompressor.h @@ -1,4 +1,10 @@ #pragma once + +// +// SPDX-License-Identifier: BSD-3-Clause +// Copyright (c) Contributors to the OpenEXR Project. +// + #include #include "ImfNamespace.h" #include "ImfCompressor.h" From d0a24d43e90f099aca7347199c6354d13b1a069c Mon Sep 17 00:00:00 2001 From: Vlad Lazar Date: Mon, 15 Jan 2024 10:15:10 -0500 Subject: [PATCH 08/34] Build Blosc as a part of the regular build (#4) Signed-off-by: Vlad-Andrei Lazar --- cmake/LibraryDefine.cmake | 11 +++++ cmake/OpenEXRSetup.cmake | 88 ++++++++++++++++++++++++++++++++++ src/lib/OpenEXR/CMakeLists.txt | 5 +- 3 files changed, 100 insertions(+), 4 deletions(-) diff --git a/cmake/LibraryDefine.cmake b/cmake/LibraryDefine.cmake index a242ec8519..8df620ca30 100644 --- a/cmake/LibraryDefine.cmake +++ b/cmake/LibraryDefine.cmake @@ -30,6 +30,17 @@ function(OPENEXR_DEFINE_LIBRARY libname) # we are embedding libdeflate target_include_directories(${objlib} PRIVATE ${EXR_DEFLATE_INCLUDE_DIR}) + # we are statically linking blosc2 + if(${objlib} STREQUAL "OpenEXR") # OR ${objlib} STREQUAL "OpenEXRCore" + target_include_directories(${objlib} PRIVATE ${BLOSC2_INCLUDE_DIRS}) + target_link_directories(${objlib} PRIVATE ${BLOSC2_LIB_DIR}) + target_link_libraries(${objlib} PRIVATE "dl" ${BLOSC2_LIB_NAME}) + if(TARGET blosc2_static) + install(TARGETS blosc2_static EXPORT ${objlib}) + endif() + endif() + + if(OPENEXR_CURLIB_PRIV_EXPORT AND BUILD_SHARED_LIBS) target_compile_definitions(${objlib} PRIVATE ${OPENEXR_CURLIB_PRIV_EXPORT}) if(WIN32) diff --git a/cmake/OpenEXRSetup.cmake b/cmake/OpenEXRSetup.cmake index 7d01362c84..c0c15174b0 100644 --- a/cmake/OpenEXRSetup.cmake +++ b/cmake/OpenEXRSetup.cmake @@ -290,6 +290,94 @@ else() endif() endif() +####################################### +# Find or install Blosc2 +####################################### + +set(MINIMUM_BLOSC2_VERSION 2.11.0) +option(OPENEXR_FORCE_INTERNAL_BLOSC2 [=[Force using installed Blosc2.]=] OFF) + +set(BLOSC2_REPO "https://github.com/Blosc/c-blosc2.git" CACHE STRING "Repo path for blosc2 source") +set(BLOSC2_TAG "v${MINIMUM_BLOSC2_VERSION}" CACHE STRING "Tag to use for blosc2 source repo") + +if(NOT OPENEXR_FORCE_INTERNAL_BLOSC2) + #TODO: ^^ Release should not clone from main, this is a place holder + set(CMAKE_IGNORE_PATH "${CMAKE_CURRENT_BINARY_DIR}/_deps/blosc2-src/config;${CMAKE_CURRENT_BINARY_DIR}/_deps/blosc2-build/config") + find_package(Blosc2 ${MINIMUM_BLOSC2_VERSION}) + set(CMAKE_IGNORE_PATH) +endif() + +if(NOT TARGET Blosc2::blosc2_static AND NOT Blosc2_FOUND) + if(OPENEXR_FORCE_INTERNAL_BLOSC2) + message(STATUS "Blosc2 forced internal, installing from ${BLOSC2_REPO} (${BLOSC2_TAG})") + else() + message(STATUS "Blosc2 was not found, installing from ${BLOSC2_REPO} (${BLOSC2_TAG})") + endif() + + # configure the blosc2 build + set(BUILD_BENCHMARKS OFF CACHE INTERNAL "no benchmarks") + set(BUILD_EXAMPLES OFF CACHE INTERNAL "no examples") + set(BUILD_FUZZERS OFF CACHE INTERNAL "no fuzzer") + set(BUILD_SHARED OFF CACHE INTERNAL "no shared library") + set(BUILD_TESTS OFF CACHE INTERNAL "no tests") + + include(FetchContent) + FetchContent_Declare(Blosc2 + GIT_REPOSITORY "${BLOSC2_REPO}" + GIT_TAG "${BLOSC2_TAG}" + GIT_SHALLOW ON + GIT_PROGRESS ON) + + FetchContent_GetProperties(Blosc2) + if(NOT Blosc2_POPULATED) + FetchContent_Populate(Blosc2) + # Propagate OpenEXR's setting for pkg-config generation to Blosc2: + # If OpenEXR is generating it, the internal Blosc2 should, too. + # TODO: Do we really need that here ? + # set(BLOSC2_INSTALL_PKG_CONFIG ${OPENEXR_INSTALL_PKG_CONFIG}) + add_subdirectory(${blosc2_SOURCE_DIR} ${blosc2_BINARY_DIR}) + endif() + # the install creates this but if we're using the library locally we + # haven't installed the header files yet, so need to extract those + # and make a variable for header only usage + if(NOT TARGET Blosc2::Blosc2Config) + # TODO: Do we really need that here ? + # get_target_property(blosc2inc blosc2_static INTERFACE_INCLUDE_DIRECTORIES) + # get_target_property(blosc2confinc blosc2_shared INTERFACE_INCLUDE_DIRECTORIES) + # list(APPEND blosc2inc ${blosc2confinc}) + # set(BLOSC2_HEADER_ONLY_INCLUDE_DIRS ${blosc2inc}) + # message(STATUS "Blosc2 interface dirs ${BLOSC2_HEADER_ONLY_INCLUDE_DIRS}") + + get_target_property(blosc2inc Blosc2::blosc2_static INCLUDE_DIRECTORIES) + set(BLOSC2_INCLUDE_DIRS ${blosc2inc}) + + get_target_property(blosc2libdir Blosc2::blosc2_static BINARY_DIR) + set(BLOSC2_LIB_DIR ${blosc2libdir}) + + get_target_property(blosc2libname Blosc2::blosc2_static OUTPUT_NAME) + set(BLOSC2_LIB_NAME ${blosc2libname}) + endif() +else() + message(STATUS "Using Blosc2 ${Blosc2_VERSION} from ${Blosc2_DIR}") + # local build + if(NOT TARGET Blosc2::Blosc2Config AND TARGET Blosc2 AND TARGET Blosc2Config) + get_target_property(blosc2inc blosc2_static INTERFACE_INCLUDE_DIRECTORIES) + get_target_property(blosc2confinc blosc2_shared INTERFACE_INCLUDE_DIRECTORIES) + list(APPEND blosc2inc ${blosc2confinc}) + set(BLOSC2_HEADER_ONLY_INCLUDE_DIRS ${blosc2inc}) + message(STATUS "Blosc2 internal interface dirs: ${BLOSC2_HEADER_ONLY_INCLUDE_DIRS}") + + get_target_property(blosc2inc Blosc2::blosc2_static INCLUDE_DIRECTORIES) + set(BLOSC2_INCLUDE_DIRS ${blosc2inc}) + + get_target_property(blosc2libdir Blosc2::blosc2_static BINARY_DIR) + set(BLOSC2_LIB_DIR ${blosc2libdir}) + + get_target_property(blosc2libname Blosc2::blosc2_static OUTPUT_NAME) + set(BLOSC2_LIB_NAME ${blosc2libname}) + endif() +endif() + ########################################### # Check if we need to emulate vld1q_f32_x2 ########################################### diff --git a/src/lib/OpenEXR/CMakeLists.txt b/src/lib/OpenEXR/CMakeLists.txt index 45a809f386..1cccbdd8be 100644 --- a/src/lib/OpenEXR/CMakeLists.txt +++ b/src/lib/OpenEXR/CMakeLists.txt @@ -217,12 +217,9 @@ openexr_define_library(OpenEXR ImfXdr.h DEPENDENCIES Imath::Imath + Blosc2::blosc2_static OpenEXR::Config OpenEXR::Iex OpenEXR::IlmThread OpenEXR::OpenEXRCore ) - -target_include_directories(OpenEXR PUBLIC "/home/vladal/bin/include/") -target_link_directories(OpenEXR PUBLIC "/home/vladal/bin/lib") -target_link_libraries(OpenEXR PUBLIC "dl" "blosc2") \ No newline at end of file From 1e6698ed3be421b2ec3edf8ba9bbf5a652f7fccb Mon Sep 17 00:00:00 2001 From: Vlad Lazar Date: Mon, 22 Jan 2024 13:50:48 -0500 Subject: [PATCH 09/34] Implement the compressor inside OpenEXRCore (#5) * Whitespaces and licensing * WIP OpenEXRCore implementation * Brand new spanking blosc build. * Switch to single Scanline zstd compression and Single implementation * Fixed the tests * Undo whitespace changes * Last touches * Revert extra build changes Signed-off-by: Vlad-Andrei Lazar --- src/lib/OpenEXR/CMakeLists.txt | 1 - src/lib/OpenEXR/ImfCompressor.cpp | 7 +- src/lib/OpenEXR/ImfMultiPartInputFile.cpp | 3 +- src/lib/OpenEXR/ImfZstdCompressor.cpp | 129 ++++------------------ src/lib/OpenEXR/ImfZstdCompressor.h | 25 +---- src/lib/OpenEXRCore/CMakeLists.txt | 4 +- src/lib/OpenEXRCore/decoding.c | 4 + src/lib/OpenEXRCore/encoding.c | 1 + src/lib/OpenEXRCore/internal_compress.h | 1 + src/lib/OpenEXRCore/internal_decompress.h | 7 ++ src/lib/OpenEXRCore/internal_zstd.c | 115 +++++++++++++++++++ src/lib/OpenEXRCore/openexr_attr.h | 1 + src/lib/OpenEXRCore/openexr_compression.h | 11 ++ src/lib/OpenEXRCore/parse_header.c | 2 + src/test/OpenEXRCoreTest/compression.cpp | 12 ++ src/test/OpenEXRCoreTest/compression.h | 2 + src/test/OpenEXRCoreTest/main.cpp | 2 + 17 files changed, 194 insertions(+), 133 deletions(-) create mode 100644 src/lib/OpenEXRCore/internal_zstd.c diff --git a/src/lib/OpenEXR/CMakeLists.txt b/src/lib/OpenEXR/CMakeLists.txt index 1cccbdd8be..c6759e7031 100644 --- a/src/lib/OpenEXR/CMakeLists.txt +++ b/src/lib/OpenEXR/CMakeLists.txt @@ -217,7 +217,6 @@ openexr_define_library(OpenEXR ImfXdr.h DEPENDENCIES Imath::Imath - Blosc2::blosc2_static OpenEXR::Config OpenEXR::Iex OpenEXR::IlmThread diff --git a/src/lib/OpenEXR/ImfCompressor.cpp b/src/lib/OpenEXR/ImfCompressor.cpp index dfb5ab8adf..db51021dc6 100644 --- a/src/lib/OpenEXR/ImfCompressor.cpp +++ b/src/lib/OpenEXR/ImfCompressor.cpp @@ -19,6 +19,7 @@ #include "ImfRleCompressor.h" #include "ImfZipCompressor.h" #include "ImfZstdCompressor.h" +#include "openexr_compression.h" OPENEXR_IMF_INTERNAL_NAMESPACE_SOURCE_ENTER @@ -145,7 +146,7 @@ newCompressor (Compression c, size_t maxScanLineSize, const Header& hdr) DwaCompressor::STATIC_HUFFMAN); case ZSTD_COMPRESSION: - return new ZstdCompressor (hdr, maxScanLineSize, 32); + return new ZstdCompressor (hdr); default: return 0; } } @@ -167,7 +168,7 @@ numLinesInBuffer (Compression comp) case B44_COMPRESSION: case B44A_COMPRESSION: case DWAA_COMPRESSION: return 32; - case ZSTD_COMPRESSION: return 32; + case ZSTD_COMPRESSION: return (int)exr_get_zstd_lines_per_chunk(); case DWAB_COMPRESSION: return 256; default: throw IEX_NAMESPACE::ArgExc ("Unknown compression type"); @@ -190,7 +191,7 @@ newTileCompressor ( return new ZipCompressor (hdr, tileLineSize, numTileLines); case ZSTD_COMPRESSION: - return new ZstdCompressor (hdr, tileLineSize, numTileLines); + return new ZstdCompressor (hdr); case PIZ_COMPRESSION: diff --git a/src/lib/OpenEXR/ImfMultiPartInputFile.cpp b/src/lib/OpenEXR/ImfMultiPartInputFile.cpp index b9ed6d5d29..b6fb9aeb12 100644 --- a/src/lib/OpenEXR/ImfMultiPartInputFile.cpp +++ b/src/lib/OpenEXR/ImfMultiPartInputFile.cpp @@ -22,6 +22,7 @@ #include "ImfTiledMisc.h" #include "ImfTimeCodeAttribute.h" #include "ImfVersion.h" +#include "openexr_compression.h" #include @@ -547,7 +548,7 @@ MultiPartInputFile::Data::chunkOffsetReconstruction ( // (TODO) fix this so that it doesn't need to be revised for future compression types. switch (parts[i]->header.compression ()) { - case ZSTD_COMPRESSION: rowsizes[i] = 32; break; + case ZSTD_COMPRESSION: rowsizes[i] = (int)exr_get_zstd_lines_per_chunk(); break; case DWAB_COMPRESSION: rowsizes[i] = 256; break; case PIZ_COMPRESSION: case B44_COMPRESSION: diff --git a/src/lib/OpenEXR/ImfZstdCompressor.cpp b/src/lib/OpenEXR/ImfZstdCompressor.cpp index d35a3deecd..55b4a8cea1 100644 --- a/src/lib/OpenEXR/ImfZstdCompressor.cpp +++ b/src/lib/OpenEXR/ImfZstdCompressor.cpp @@ -4,75 +4,42 @@ // #include +#include +#include "openexr_compression.h" #include "ImfZstdCompressor.h" -#include "blosc2.h" #include "IlmThreadPool.h" #include "ImfChannelList.h" #include "ImfMisc.h" + namespace { -class BloscInit -{ -public: - static void Init () { getInstance (); } - BloscInit (const BloscInit&) = delete; - BloscInit& operator= (const BloscInit&) = delete; - -private: - BloscInit () { blosc2_init (); } - ~BloscInit () { blosc2_destroy (); } - static BloscInit& getInstance () - { - static BloscInit instance; - return instance; - } -}; -} // namespace - +std::mutex g_mutex; +} + OPENEXR_IMF_INTERNAL_NAMESPACE_SOURCE_ENTER -ZstdCompressor::ZstdCompressor ( - const Header& hdr, size_t maxScanlineSize, size_t numScanLines) - : Compressor (hdr) - , _maxScanlineSize (maxScanlineSize) - , _numScanLines (numScanLines) - , _outBuffer (nullptr, &free) - , _schunk (nullptr, &blosc2_schunk_free) + +ZstdCompressor::ZstdCompressor (const Header& hdr) + : Compressor (hdr), _outBuffer () {} int ZstdCompressor::numScanLines () const { - return _numScanLines; // Needs to be in sync with ImfCompressor::numLinesInBuffer + return (int)exr_get_zstd_lines_per_chunk(); // Needs to be in sync with ImfCompressor::numLinesInBuffer } int ZstdCompressor::compress ( const char* inPtr, int inSize, int minY, const char*& outPtr) { - int typeSize = std::numeric_limits::min (); - for (auto it = header ().channels ().begin (); - it != header ().channels ().end (); - ++it) + outPtr = (char*) malloc (inSize); { - // BLOSC prefilter is affected by the typesize. Initializing to max will ensure that a channel is not split in 2 and filtered separately. - // probably compression can be improved for non-deep images by compressing every channel separately with the correct typeSize - // (much harder to do for Deep-Data). - typeSize = std::max (typeSize, Imf::pixelTypeSize (it.channel ().type)); + std::lock_guard lock (g_mutex); + _outBuffer.push_back (raw_ptr ((char*) outPtr, &free)); } - - auto ret = BLOSC_compress_impl (inPtr, inSize, typeSize, outPtr); - auto fullSize = Xdr::size () + ret; - auto data = malloc (fullSize); - auto write = (char*) data; - - Xdr::write (write, Versions::LATEST); - - memcpy (write, outPtr, ret); - outPtr = (char*) data; - - _outBuffer = raw_ptr ((char*) data, &free); - + auto fullSize = + exr_compress_zstd ((char*) (inPtr), inSize, (void*) outPtr, inSize); return fullSize; } @@ -80,67 +47,15 @@ int ZstdCompressor::uncompress ( const char* inPtr, int inSize, int minY, const char*& outPtr) { - auto read = (const char*) inPtr; - int v; - Xdr::read (read, v); - if (v == Versions::SINGLE_BLOB) + auto read = (const char*) inPtr; + void* write = nullptr; + auto ret = exr_uncompress_zstd (read, inSize, &write, 0); { - return BLOSC_uncompress_impl_single_blob ( - read, inSize - Xdr::size (), outPtr); + std::lock_guard lock (g_mutex); + _outBuffer.push_back (raw_ptr ((char*) write, &free)); } - else { throw Iex::InputExc ("Unsupported ZstdCompressor version"); } -} - -int -ZstdCompressor::BLOSC_compress_impl ( - const char* inPtr, int inSize, int typeSize, const char*& out) -{ - BloscInit::Init (); - blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS; - - cparams.typesize = typeSize; - // clevel 9 is about a 20% increase in compression compared to 5. - // Decompression speed is unchanged. - cparams.clevel = header ().zstdCompressionLevel (); - cparams.nthreads = 1; - cparams.compcode = BLOSC_ZSTD; // Codec - cparams.splitmode = - BLOSC_NEVER_SPLIT; // Split => multithreading, not split better compression - - blosc2_storage storage = BLOSC2_STORAGE_DEFAULTS; - storage.cparams = &cparams; - storage.contiguous = true; - - _schunk = schunk_ptr (blosc2_schunk_new (&storage), &blosc2_schunk_free); - - auto in = const_cast (inPtr); - blosc2_schunk_append_buffer (_schunk.get (), in, inSize); - - uint8_t* buffer; - bool shouldFree = true; - auto size = blosc2_schunk_to_buffer (_schunk.get (), &buffer, &shouldFree); - out = (char*) buffer; - if (shouldFree) { _outBuffer = raw_ptr ((char*) buffer, &free); } - return size; -} - -int -ZstdCompressor::BLOSC_uncompress_impl_single_blob ( - const char* inPtr, int inSize, const char*& out) -{ - auto in = const_cast (inPtr); - _schunk = schunk_ptr ( - blosc2_schunk_from_buffer ( - reinterpret_cast (in), inSize, true), - &blosc2_schunk_free); - - auto buffSize = _maxScanlineSize * numScanLines (); - _outBuffer = - Imf::ZstdCompressor::raw_ptr ((char*) malloc (buffSize), &free); - auto size = blosc2_schunk_decompress_chunk ( - _schunk.get (), 0, _outBuffer.get (), buffSize); - out = _outBuffer.get (); - return size; + outPtr = (const char*) write; + return ret; } OPENEXR_IMF_INTERNAL_NAMESPACE_SOURCE_EXIT \ No newline at end of file diff --git a/src/lib/OpenEXR/ImfZstdCompressor.h b/src/lib/OpenEXR/ImfZstdCompressor.h index d9319e076c..c9f045cddd 100644 --- a/src/lib/OpenEXR/ImfZstdCompressor.h +++ b/src/lib/OpenEXR/ImfZstdCompressor.h @@ -1,45 +1,30 @@ -#pragma once - // // SPDX-License-Identifier: BSD-3-Clause // Copyright (c) Contributors to the OpenEXR Project. // +#pragma once + #include #include "ImfNamespace.h" #include "ImfCompressor.h" #include "ImfHeader.h" #include "blosc2.h" +#include "vector" OPENEXR_IMF_INTERNAL_NAMESPACE_HEADER_ENTER class ZstdCompressor : public Compressor { public: - ZstdCompressor ( - const Header& hdr, size_t maxScanLines, size_t numScanLines); - + explicit ZstdCompressor (const Header& hdr); private: - using schunk_ptr = - std::unique_ptr; using raw_ptr = std::unique_ptr; - raw_ptr _outBuffer; - schunk_ptr _schunk; - size_t _maxScanlineSize; - size_t _numScanLines; + std::vector _outBuffer; int numScanLines () const override; // max int compress ( const char* inPtr, int inSize, int minY, const char*& outPtr) override; int uncompress ( const char* inPtr, int inSize, int minY, const char*& outPtr) override; - int BLOSC_compress_impl ( - const char* inPtr, int inSize, int typeSize, const char*& out); - int BLOSC_uncompress_impl_single_blob ( - const char* inPtr, int inSize, const char*& out); - enum Versions : int - { - SINGLE_BLOB = 1, - LATEST = SINGLE_BLOB - }; }; OPENEXR_IMF_INTERNAL_NAMESPACE_HEADER_EXIT \ No newline at end of file diff --git a/src/lib/OpenEXRCore/CMakeLists.txt b/src/lib/OpenEXRCore/CMakeLists.txt index f4bd54efb3..ecfc171202 100644 --- a/src/lib/OpenEXRCore/CMakeLists.txt +++ b/src/lib/OpenEXRCore/CMakeLists.txt @@ -45,6 +45,7 @@ openexr_define_library(OpenEXRCore internal_piz.c internal_dwa.c internal_huf.c + internal_zstd.c attributes.c string.c @@ -102,6 +103,7 @@ openexr_define_library(OpenEXRCore DEPENDENCIES Imath::Imath + Blosc2::blosc2_static ) if (DEFINED EXR_DEFLATE_LIB) @@ -110,4 +112,4 @@ if (DEFINED EXR_DEFLATE_LIB) else() target_link_libraries(OpenEXRCore PUBLIC ${EXR_DEFLATE_LIB}) endif() -endif() +endif() \ No newline at end of file diff --git a/src/lib/OpenEXRCore/decoding.c b/src/lib/OpenEXRCore/decoding.c index 322cbd8965..c7db79f755 100644 --- a/src/lib/OpenEXRCore/decoding.c +++ b/src/lib/OpenEXRCore/decoding.c @@ -263,6 +263,10 @@ decompress_data ( rv = internal_exr_undo_dwab ( decode, packbufptr, packsz, unpackbufptr, unpacksz); break; + case EXR_COMPRESSION_ZSTD: + rv = internal_exr_undo_zstd ( + decode, packbufptr, packsz, unpackbufptr, unpacksz); + break; case EXR_COMPRESSION_LAST_TYPE: default: return pctxt->print_error ( diff --git a/src/lib/OpenEXRCore/encoding.c b/src/lib/OpenEXRCore/encoding.c index b92e0ce356..6b017ed7b0 100644 --- a/src/lib/OpenEXRCore/encoding.c +++ b/src/lib/OpenEXRCore/encoding.c @@ -54,6 +54,7 @@ default_compress_chunk (exr_encode_pipeline_t* encode) case EXR_COMPRESSION_B44A: rv = internal_exr_apply_b44a (encode); break; case EXR_COMPRESSION_DWAA: rv = internal_exr_apply_dwaa (encode); break; case EXR_COMPRESSION_DWAB: rv = internal_exr_apply_dwab (encode); break; + case EXR_COMPRESSION_ZSTD: rv = internal_exr_apply_zstd (encode); break; case EXR_COMPRESSION_LAST_TYPE: default: return pctxt->print_error ( diff --git a/src/lib/OpenEXRCore/internal_compress.h b/src/lib/OpenEXRCore/internal_compress.h index 360015c120..27315f68a0 100644 --- a/src/lib/OpenEXRCore/internal_compress.h +++ b/src/lib/OpenEXRCore/internal_compress.h @@ -33,4 +33,5 @@ exr_result_t internal_exr_apply_dwaa (exr_encode_pipeline_t* encode); exr_result_t internal_exr_apply_dwab (exr_encode_pipeline_t* encode); +exr_result_t internal_exr_apply_zstd (exr_encode_pipeline_t* encode); #endif /* OPENEXR_CORE_COMPRESS_H */ diff --git a/src/lib/OpenEXRCore/internal_decompress.h b/src/lib/OpenEXRCore/internal_decompress.h index 834b854a3e..2726a4afd3 100644 --- a/src/lib/OpenEXRCore/internal_decompress.h +++ b/src/lib/OpenEXRCore/internal_decompress.h @@ -73,4 +73,11 @@ exr_result_t internal_exr_undo_dwab ( void* uncompressed_data, uint64_t uncompressed_size); +exr_result_t internal_exr_undo_zstd ( + exr_decode_pipeline_t* decode, + const void* compressed_data, + uint64_t comp_buf_size, + void* uncompressed_data, + uint64_t uncompressed_size); + #endif /* OPENEXR_CORE_DECOMPRESS_H */ diff --git a/src/lib/OpenEXRCore/internal_zstd.c b/src/lib/OpenEXRCore/internal_zstd.c new file mode 100644 index 0000000000..1fc845c6a0 --- /dev/null +++ b/src/lib/OpenEXRCore/internal_zstd.c @@ -0,0 +1,115 @@ +/* +** SPDX-License-Identifier: BSD-3-Clause +** Copyright Contributors to the OpenEXR Project. +*/ + +#include +#include "internal_compress.h" +#include "internal_decompress.h" +#include "blosc2.h" + +size_t +exr_get_zstd_lines_per_chunk () +{ + return 1; +} + +long +exr_compress_zstd (char* inPtr, int inSize, void* outPtr, int outPtrSize) +{ + if (inSize == 0) // Weird input data when subsampling + { + outPtr = NULL; + return 0; + } + + blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS; + int typeSize = inSize % 4 == 0 ? 4 : 2; + cparams.typesize = typeSize; + // clevel 9 is about a 20% increase in compression compared to 5. + // Decompression speed is unchanged. + int zstd_level; + exr_get_default_zstd_compression_level (&zstd_level); + cparams.clevel = zstd_level; + cparams.nthreads = 1; + cparams.compcode = BLOSC_ZSTD; // Codec + cparams.splitmode = + BLOSC_NEVER_SPLIT; // Split => multithreading, not split better compression + + blosc2_storage storage = BLOSC2_STORAGE_DEFAULTS; + storage.contiguous = true; + storage.cparams = &cparams; + + blosc2_schunk* _schunk = blosc2_schunk_new (&storage); + + blosc2_schunk_append_buffer (_schunk, inPtr, inSize); + + uint8_t* buffer; + bool shouldFree = true; + int64_t size = blosc2_schunk_to_buffer (_schunk, &buffer, &shouldFree); + + if (size <= inSize && size <= outPtrSize && size > 0) + { memcpy (outPtr, buffer, size); } + else + { + memcpy (outPtr, inPtr, inSize); + size = inSize; // We increased compression size + } + + if (shouldFree) { free (buffer); } + + blosc2_schunk_free (_schunk); + return size; +} + +long +exr_uncompress_zstd ( + const char* inPtr, uint64_t inSize, void** outPtr, uint64_t outPtrSize) +{ + blosc2_schunk* _schunk = blosc2_schunk_from_buffer ((uint8_t *)inPtr, inSize, true); + + if (_schunk == NULL) { return -1; } + + if (outPtrSize == 0) // we don't have any storage allocated + { + *outPtr = malloc (_schunk->nbytes); + outPtrSize = _schunk->nbytes; + } + + int size = blosc2_schunk_decompress_chunk (_schunk, 0, *outPtr, outPtrSize); + blosc2_schunk_free (_schunk); + + return size; +} + +exr_result_t +internal_exr_apply_zstd (exr_encode_pipeline_t* encode) +{ + long compressedSize = exr_compress_zstd ( + encode->packed_buffer, + encode->packed_bytes, + encode->compressed_buffer, + encode->compressed_alloc_size); + if (compressedSize < 0) { return EXR_ERR_UNKNOWN; } + + encode->compressed_bytes = compressedSize; + return EXR_ERR_SUCCESS; +} + +exr_result_t +internal_exr_undo_zstd ( + exr_decode_pipeline_t* decode, + const void* compressed_data, + uint64_t comp_buf_size, + void* uncompressed_data, + uint64_t uncompressed_size) +{ + + long uncompressedSize = exr_uncompress_zstd ( + (const char*) compressed_data, + comp_buf_size, + &uncompressed_data, + uncompressed_size); + if (uncompressed_size != uncompressedSize) { return EXR_ERR_CORRUPT_CHUNK; } + return EXR_ERR_SUCCESS; +} \ No newline at end of file diff --git a/src/lib/OpenEXRCore/openexr_attr.h b/src/lib/OpenEXRCore/openexr_attr.h index eabcd57d9c..e6aa7421ba 100644 --- a/src/lib/OpenEXRCore/openexr_attr.h +++ b/src/lib/OpenEXRCore/openexr_attr.h @@ -45,6 +45,7 @@ typedef enum EXR_COMPRESSION_B44A = 7, EXR_COMPRESSION_DWAA = 8, EXR_COMPRESSION_DWAB = 9, + EXR_COMPRESSION_ZSTD = 10, EXR_COMPRESSION_LAST_TYPE /**< Invalid value, provided for range checking. */ } exr_compression_t; diff --git a/src/lib/OpenEXRCore/openexr_compression.h b/src/lib/OpenEXRCore/openexr_compression.h index 67ae45004b..e5956cfc85 100644 --- a/src/lib/OpenEXRCore/openexr_compression.h +++ b/src/lib/OpenEXRCore/openexr_compression.h @@ -45,6 +45,17 @@ exr_result_t exr_uncompress_buffer ( size_t out_bytes_avail, size_t* actual_out); +EXR_EXPORT +long exr_compress_zstd ( + char* inPtr, int inSize, void * outPtr, int outPtrSize); + +EXR_EXPORT +long exr_uncompress_zstd ( + const char* inPtr, uint64_t inSize, void ** outPtr, uint64_t outPtrSize); + +EXR_EXPORT +size_t exr_get_zstd_lines_per_chunk(); + #ifdef __cplusplus } /* extern "C" */ #endif diff --git a/src/lib/OpenEXRCore/parse_header.c b/src/lib/OpenEXRCore/parse_header.c index b85273a378..a7acc4621d 100644 --- a/src/lib/OpenEXRCore/parse_header.c +++ b/src/lib/OpenEXRCore/parse_header.c @@ -9,6 +9,7 @@ #include "internal_constants.h" #include "internal_structs.h" #include "internal_xdr.h" +#include "openexr_compression.h" #include #include @@ -2364,6 +2365,7 @@ internal_exr_compute_chunk_offset_size (struct _internal_exr_part* curpart) case EXR_COMPRESSION_B44A: case EXR_COMPRESSION_DWAA: linePerChunk = 32; break; case EXR_COMPRESSION_DWAB: linePerChunk = 256; break; + case EXR_COMPRESSION_ZSTD: linePerChunk = exr_get_zstd_lines_per_chunk(); break; case EXR_COMPRESSION_LAST_TYPE: default: /* ERROR CONDITION */ diff --git a/src/test/OpenEXRCoreTest/compression.cpp b/src/test/OpenEXRCoreTest/compression.cpp index 772111647f..e0764e9f0b 100644 --- a/src/test/OpenEXRCoreTest/compression.cpp +++ b/src/test/OpenEXRCoreTest/compression.cpp @@ -1419,6 +1419,7 @@ doWriteRead ( case EXR_COMPRESSION_RLE: case EXR_COMPRESSION_ZIP: case EXR_COMPRESSION_ZIPS: + case EXR_COMPRESSION_ZSTD: restore.compareExact (p, "orig", "C loaded C"); break; case EXR_COMPRESSION_PIZ: @@ -1681,6 +1682,12 @@ testDWABCompression (const std::string& tempdir) testComp (tempdir, EXR_COMPRESSION_DWAB); } +void +testZstdCompression (const std::string& tempdir) +{ + testComp (tempdir, EXR_COMPRESSION_ZSTD); +} + void testDeepNoCompression (const std::string& tempdir) {} @@ -1692,3 +1699,8 @@ testDeepZIPCompression (const std::string& tempdir) void testDeepZIPSCompression (const std::string& tempdir) {} + +void +testDeepZstdCompression (const std::string& tempdir) +{ +} \ No newline at end of file diff --git a/src/test/OpenEXRCoreTest/compression.h b/src/test/OpenEXRCoreTest/compression.h index 573e10f96c..ef5391a92b 100644 --- a/src/test/OpenEXRCoreTest/compression.h +++ b/src/test/OpenEXRCoreTest/compression.h @@ -18,9 +18,11 @@ void testB44Compression (const std::string& tempdir); void testB44ACompression (const std::string& tempdir); void testDWAACompression (const std::string& tempdir); void testDWABCompression (const std::string& tempdir); +void testZstdCompression (const std::string& tempdir); void testDeepNoCompression (const std::string& tempdir); void testDeepZIPCompression (const std::string& tempdir); void testDeepZIPSCompression (const std::string& tempdir); +void testDeepZstdCompression (const std::string& tempdir); #endif // OPENEXR_CORE_TEST_COMPRESSION_H diff --git a/src/test/OpenEXRCoreTest/main.cpp b/src/test/OpenEXRCoreTest/main.cpp index d12d6718f7..dae1ea29ae 100644 --- a/src/test/OpenEXRCoreTest/main.cpp +++ b/src/test/OpenEXRCoreTest/main.cpp @@ -202,10 +202,12 @@ main (int argc, char* argv[]) TEST (testB44ACompression, "core_compression"); TEST (testDWAACompression, "core_compression"); TEST (testDWABCompression, "core_compression"); + TEST (testZstdCompression, "core_compression"); TEST (testDeepNoCompression, "core_compression"); TEST (testDeepZIPCompression, "core_compression"); TEST (testDeepZIPSCompression, "core_compression"); + TEST (testDeepZstdCompression, "core_compression"); // empty dummy test if (helpMode) { From ccac098efda46e4774d28406d34daf90d96090e6 Mon Sep 17 00:00:00 2001 From: Philippe Leprince Date: Wed, 13 Mar 2024 12:23:05 +0100 Subject: [PATCH 10/34] Adopt new compression API Signed-off-by: Philippe Leprince --- src/lib/OpenEXR/ImfCRgbaFile.h | 3 ++- src/lib/OpenEXR/ImfCompression.cpp | 7 +++++++ src/lib/OpenEXR/ImfCompression.h | 3 ++- src/lib/OpenEXR/ImfCompressor.cpp | 9 ++++++--- src/lib/OpenEXR/ImfZstdCompressor.cpp | 5 +++-- src/lib/OpenEXR/ImfZstdCompressor.h | 10 ++++++---- 6 files changed, 26 insertions(+), 11 deletions(-) diff --git a/src/lib/OpenEXR/ImfCRgbaFile.h b/src/lib/OpenEXR/ImfCRgbaFile.h index 062f2f709a..ecdb65e59d 100644 --- a/src/lib/OpenEXR/ImfCRgbaFile.h +++ b/src/lib/OpenEXR/ImfCRgbaFile.h @@ -80,7 +80,8 @@ typedef struct ImfRgba ImfRgba; #define IMF_B44A_COMPRESSION 7 #define IMF_DWAA_COMPRESSION 8 #define IMF_DWAB_COMPRESSION 9 -#define IMF_NUM_COMPRESSION_METHODS 10 +#define IMF_ZSTD_COMPRESSION 10 +#define IMF_NUM_COMPRESSION_METHODS 11 /* ** Channels; values must be the same as in Imf::RgbaChannels. diff --git a/src/lib/OpenEXR/ImfCompression.cpp b/src/lib/OpenEXR/ImfCompression.cpp index 97631779c5..2d568fe627 100644 --- a/src/lib/OpenEXR/ImfCompression.cpp +++ b/src/lib/OpenEXR/ImfCompression.cpp @@ -98,6 +98,12 @@ static const CompressionDesc IdToDesc[] = { 256, true, false), + CompressionDesc ( + "zstd", + "blosc zstd lossless compression, one scan line at a time.", + 1, + false, + true), }; // clang-format on @@ -114,6 +120,7 @@ static const std::map CompressionNameToId = { {"b44a", Compression::B44A_COMPRESSION}, {"dwaa", Compression::DWAA_COMPRESSION}, {"dwab", Compression::DWAB_COMPRESSION}, + {"zstd", Compression::ZSTD_COMPRESSION}, }; #define UNKNOWN_COMPRESSION_ID_MSG "INVALID COMPRESSION ID" diff --git a/src/lib/OpenEXR/ImfCompression.h b/src/lib/OpenEXR/ImfCompression.h index 1ccffe5fd0..8ddd94ca9e 100644 --- a/src/lib/OpenEXR/ImfCompression.h +++ b/src/lib/OpenEXR/ImfCompression.h @@ -51,7 +51,8 @@ enum IMF_EXPORT_ENUM Compression // wise and faster to decode full frames // than DWAA_COMPRESSION. - ZSTD_COMPRESSION = 10, + ZSTD_COMPRESSION = 10, // blosc zstd lossless compression, one scan line + // at a time. NUM_COMPRESSION_METHODS // number of different compression methods. }; diff --git a/src/lib/OpenEXR/ImfCompressor.cpp b/src/lib/OpenEXR/ImfCompressor.cpp index 147de52683..94fcaecacd 100644 --- a/src/lib/OpenEXR/ImfCompressor.cpp +++ b/src/lib/OpenEXR/ImfCompressor.cpp @@ -102,7 +102,9 @@ newCompressor (Compression c, size_t maxScanLineSize, const Header& hdr) DwaCompressor::STATIC_HUFFMAN); case ZSTD_COMPRESSION: + return new ZstdCompressor (hdr); + default: return 0; } // clang-format on @@ -135,9 +137,6 @@ newTileCompressor ( case ZIP_COMPRESSION: return new ZipCompressor (hdr, tileLineSize, numTileLines); - case ZSTD_COMPRESSION: - - return new ZstdCompressor (hdr); case PIZ_COMPRESSION: @@ -171,6 +170,10 @@ newTileCompressor ( static_cast (numTileLines), DwaCompressor::STATIC_HUFFMAN); + case ZSTD_COMPRESSION: + + return new ZstdCompressor (hdr); + default: return 0; } // clang-format on diff --git a/src/lib/OpenEXR/ImfZstdCompressor.cpp b/src/lib/OpenEXR/ImfZstdCompressor.cpp index 55b4a8cea1..a0b111afb5 100644 --- a/src/lib/OpenEXR/ImfZstdCompressor.cpp +++ b/src/lib/OpenEXR/ImfZstdCompressor.cpp @@ -16,7 +16,7 @@ namespace { std::mutex g_mutex; } - + OPENEXR_IMF_INTERNAL_NAMESPACE_SOURCE_ENTER ZstdCompressor::ZstdCompressor (const Header& hdr) @@ -26,7 +26,8 @@ ZstdCompressor::ZstdCompressor (const Header& hdr) int ZstdCompressor::numScanLines () const { - return (int)exr_get_zstd_lines_per_chunk(); // Needs to be in sync with ImfCompressor::numLinesInBuffer + // Needs to be in sync with ImfCompressor::numLinesInBuffer + return (int) exr_get_zstd_lines_per_chunk (); } int diff --git a/src/lib/OpenEXR/ImfZstdCompressor.h b/src/lib/OpenEXR/ImfZstdCompressor.h index c9f045cddd..50c22e7dc8 100644 --- a/src/lib/OpenEXR/ImfZstdCompressor.h +++ b/src/lib/OpenEXR/ImfZstdCompressor.h @@ -13,16 +13,18 @@ #include "vector" OPENEXR_IMF_INTERNAL_NAMESPACE_HEADER_ENTER + class ZstdCompressor : public Compressor { public: explicit ZstdCompressor (const Header& hdr); + private: using raw_ptr = std::unique_ptr; - std::vector _outBuffer; - int numScanLines () const override; // max - int compress ( - const char* inPtr, int inSize, int minY, const char*& outPtr) override; + std::vector _outBuffer; + int numScanLines () const override; // max + int compress ( + const char* inPtr, int inSize, int minY, const char*& outPtr) override; int uncompress ( const char* inPtr, int inSize, int minY, const char*& outPtr) override; }; From 2341fb40da1838e79acb40c74a5249a1a701b58f Mon Sep 17 00:00:00 2001 From: Philippe Leprince Date: Wed, 13 Mar 2024 13:19:34 +0100 Subject: [PATCH 11/34] Fix API tests Signed-off-by: Philippe Leprince --- src/test/OpenEXRTest/testCompressionApi.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/test/OpenEXRTest/testCompressionApi.cpp b/src/test/OpenEXRTest/testCompressionApi.cpp index 53b6ecd265..7736465acd 100644 --- a/src/test/OpenEXRTest/testCompressionApi.cpp +++ b/src/test/OpenEXRTest/testCompressionApi.cpp @@ -28,11 +28,11 @@ testCompressionApi (const string& tempDir) cout << "Testing compression API functions." << endl; // update this if you add a new compressor. - string codecList = "none/rle/zips/zip/piz/pxr24/b44/b44a/dwaa/dwab"; + string codecList = "none/rle/zips/zip/piz/pxr24/b44/b44a/dwaa/dwab/zstd"; int numMethods = static_cast (NUM_COMPRESSION_METHODS); // update this if you add a new compressor. - assert (numMethods == 10); + assert (numMethods == 11); for (int i = 0; i < numMethods; i++) { @@ -64,6 +64,7 @@ testCompressionApi (const string& tempDir) case ZIPS_COMPRESSION: case ZIP_COMPRESSION: case PIZ_COMPRESSION: + case ZSTD_COMPRESSION: assert (isLossyCompression (c) == false); break; @@ -76,6 +77,7 @@ testCompressionApi (const string& tempDir) case NO_COMPRESSION: case RLE_COMPRESSION: case ZIPS_COMPRESSION: + case ZSTD_COMPRESSION: assert (isValidDeepCompression (c) == true); break; From 6cfc3cc0afa75fa7354112321b215d81b11b87ea Mon Sep 17 00:00:00 2001 From: Philippe Leprince Date: Wed, 13 Mar 2024 18:13:06 +0100 Subject: [PATCH 12/34] Fix cmake build Signed-off-by: Philippe Leprince --- cmake/LibraryDefine.cmake | 6 +++- cmake/OpenEXRSetup.cmake | 58 +++++++++++++++++++++++---------------- 2 files changed, 40 insertions(+), 24 deletions(-) diff --git a/cmake/LibraryDefine.cmake b/cmake/LibraryDefine.cmake index 14653430f3..541d48aefd 100644 --- a/cmake/LibraryDefine.cmake +++ b/cmake/LibraryDefine.cmake @@ -31,7 +31,11 @@ function(OPENEXR_DEFINE_LIBRARY libname) target_include_directories(${objlib} PRIVATE ${EXR_DEFLATE_INCLUDE_DIR}) # we are statically linking blosc2 - if(${objlib} STREQUAL "OpenEXR") # OR ${objlib} STREQUAL "OpenEXRCore" + if(${objlib} STREQUAL "OpenEXR" OR ${objlib} STREQUAL "OpenEXRCore") + message(STATUS "Blosc2: setting up for ${objlib}...") + message(STATUS ">> BLOSC2_INCLUDE_DIRS: ${BLOSC2_INCLUDE_DIRS}") + message(STATUS ">> BLOSC2_LIB_DIR: ${BLOSC2_LIB_DIR}") + message(STATUS ">> BLOSC2_LIB_NAME: ${BLOSC2_LIB_NAME}") target_include_directories(${objlib} PRIVATE ${BLOSC2_INCLUDE_DIRS}) target_link_directories(${objlib} PRIVATE ${BLOSC2_LIB_DIR}) target_link_libraries(${objlib} PRIVATE "dl" ${BLOSC2_LIB_NAME}) diff --git a/cmake/OpenEXRSetup.cmake b/cmake/OpenEXRSetup.cmake index 46af170e30..48335a582a 100644 --- a/cmake/OpenEXRSetup.cmake +++ b/cmake/OpenEXRSetup.cmake @@ -1,6 +1,21 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) Contributors to the OpenEXR Project. +function(_error_if_not_found prop var fallback) + message(STATUS "Blosc2: ${prop} ${var} '${fallback}'") + string(FIND "${var}" "-NOTFOUND" pos) + if(NOT pos EQUAL -1) + if(fallback STREQUAL "") + message(FATAL_ERROR "Blosc2: Property ${prop} not found: ${var}") + else() + string(SUBSTRING "${var}" 0 ${pos} var_name) + message(STATUS "Blosc2: Property ${prop} not found: ${var_name} falling back to '${fallback}'") + set(${var_name} "${fallback}" PARENT_SCOPE) + endif() + endif() +endfunction(_error_if_not_found) + + include(GNUInstallDirs) if(NOT "${CMAKE_PROJECT_NAME}" STREQUAL "${PROJECT_NAME}") @@ -334,18 +349,20 @@ option(OPENEXR_FORCE_INTERNAL_BLOSC2 [=[Force using installed Blosc2.]=] OFF) set(BLOSC2_REPO "https://github.com/Blosc/c-blosc2.git" CACHE STRING "Repo path for blosc2 source") set(BLOSC2_TAG "v${MINIMUM_BLOSC2_VERSION}" CACHE STRING "Tag to use for blosc2 source repo") +# Try to find a local bloc2 install if allowed to. if(NOT OPENEXR_FORCE_INTERNAL_BLOSC2) - #TODO: ^^ Release should not clone from main, this is a place holder + message(STATUS "Blosc2: Looking for local install...") set(CMAKE_IGNORE_PATH "${CMAKE_CURRENT_BINARY_DIR}/_deps/blosc2-src/config;${CMAKE_CURRENT_BINARY_DIR}/_deps/blosc2-build/config") find_package(Blosc2 ${MINIMUM_BLOSC2_VERSION}) set(CMAKE_IGNORE_PATH) endif() if(NOT TARGET Blosc2::blosc2_static AND NOT Blosc2_FOUND) + # we didn't find a local install: let's get it from its repository. if(OPENEXR_FORCE_INTERNAL_BLOSC2) - message(STATUS "Blosc2 forced internal, installing from ${BLOSC2_REPO} (${BLOSC2_TAG})") + message(STATUS "Blosc2: forced internal, installing from ${BLOSC2_REPO} (${BLOSC2_TAG})") else() - message(STATUS "Blosc2 was not found, installing from ${BLOSC2_REPO} (${BLOSC2_TAG})") + message(STATUS "Blosc2: no local blosc2 found, installing from ${BLOSC2_REPO} (${BLOSC2_TAG})") endif() # configure the blosc2 build @@ -364,23 +381,18 @@ if(NOT TARGET Blosc2::blosc2_static AND NOT Blosc2_FOUND) FetchContent_GetProperties(Blosc2) if(NOT Blosc2_POPULATED) + message(STATUS "Blosc2: Downloading ${BLOSC2_TAG} from ${BLOSC2_REPO}...") FetchContent_Populate(Blosc2) - # Propagate OpenEXR's setting for pkg-config generation to Blosc2: - # If OpenEXR is generating it, the internal Blosc2 should, too. - # TODO: Do we really need that here ? - # set(BLOSC2_INSTALL_PKG_CONFIG ${OPENEXR_INSTALL_PKG_CONFIG}) add_subdirectory(${blosc2_SOURCE_DIR} ${blosc2_BINARY_DIR}) + else() + message(STATUS "Blosc2: repo code has already been downloaded.") endif() + # the install creates this but if we're using the library locally we # haven't installed the header files yet, so need to extract those # and make a variable for header only usage - if(NOT TARGET Blosc2::Blosc2Config) - # TODO: Do we really need that here ? - # get_target_property(blosc2inc blosc2_static INTERFACE_INCLUDE_DIRECTORIES) - # get_target_property(blosc2confinc blosc2_shared INTERFACE_INCLUDE_DIRECTORIES) - # list(APPEND blosc2inc ${blosc2confinc}) - # set(BLOSC2_HEADER_ONLY_INCLUDE_DIRS ${blosc2inc}) - # message(STATUS "Blosc2 interface dirs ${BLOSC2_HEADER_ONLY_INCLUDE_DIRS}") + if(TARGET Blosc2::blosc2_static) + message(STATUS "Blosc2: Setting up blosc directories") get_target_property(blosc2inc Blosc2::blosc2_static INCLUDE_DIRECTORIES) set(BLOSC2_INCLUDE_DIRS ${blosc2inc}) @@ -392,22 +404,22 @@ if(NOT TARGET Blosc2::blosc2_static AND NOT Blosc2_FOUND) set(BLOSC2_LIB_NAME ${blosc2libname}) endif() else() - message(STATUS "Using Blosc2 ${Blosc2_VERSION} from ${Blosc2_DIR}") + message(STATUS "Blosc2: Using installed Blosc2 ${Blosc2_VERSION} from ${Blosc2_DIR}") # local build - if(NOT TARGET Blosc2::Blosc2Config AND TARGET Blosc2 AND TARGET Blosc2Config) - get_target_property(blosc2inc blosc2_static INTERFACE_INCLUDE_DIRECTORIES) - get_target_property(blosc2confinc blosc2_shared INTERFACE_INCLUDE_DIRECTORIES) - list(APPEND blosc2inc ${blosc2confinc}) - set(BLOSC2_HEADER_ONLY_INCLUDE_DIRS ${blosc2inc}) - message(STATUS "Blosc2 internal interface dirs: ${BLOSC2_HEADER_ONLY_INCLUDE_DIRS}") - - get_target_property(blosc2inc Blosc2::blosc2_static INCLUDE_DIRECTORIES) + if(TARGET Blosc2::blosc2_static) + message(STATUS "Blosc2: Setting up installed blosc directories") + + get_target_property(blosc2inc Blosc2::blosc2_static INTERFACE_INCLUDE_DIRECTORIES) + _error_if_not_found("INTERFACE_INCLUDE_DIRECTORIES" ${blosc2inc} "") set(BLOSC2_INCLUDE_DIRS ${blosc2inc}) get_target_property(blosc2libdir Blosc2::blosc2_static BINARY_DIR) + _error_if_not_found("BINARY_DIR" ${blosc2libdir} "") set(BLOSC2_LIB_DIR ${blosc2libdir}) get_target_property(blosc2libname Blosc2::blosc2_static OUTPUT_NAME) + # fallback because it fails for me but I think it may be a bug. + _error_if_not_found("OUTPUT_NAME" ${blosc2libname} "blosc2") set(BLOSC2_LIB_NAME ${blosc2libname}) endif() endif() From 88c034843926271dabdb30641b59e2e8d855d9e2 Mon Sep 17 00:00:00 2001 From: Philippe Leprince Date: Tue, 2 Apr 2024 15:09:06 +0200 Subject: [PATCH 13/34] fix: debug build was not linking against debug blosc lib. Signed-off-by: Philippe Leprince --- cmake/OpenEXRSetup.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/OpenEXRSetup.cmake b/cmake/OpenEXRSetup.cmake index 48335a582a..064ec8e4bc 100644 --- a/cmake/OpenEXRSetup.cmake +++ b/cmake/OpenEXRSetup.cmake @@ -401,7 +401,7 @@ if(NOT TARGET Blosc2::blosc2_static AND NOT Blosc2_FOUND) set(BLOSC2_LIB_DIR ${blosc2libdir}) get_target_property(blosc2libname Blosc2::blosc2_static OUTPUT_NAME) - set(BLOSC2_LIB_NAME ${blosc2libname}) + set(BLOSC2_LIB_NAME $, "${blosc2libname}${CMAKE_DEBUG_POSTFIX}", "${blosc2libname}">) endif() else() message(STATUS "Blosc2: Using installed Blosc2 ${Blosc2_VERSION} from ${Blosc2_DIR}") @@ -420,7 +420,7 @@ else() get_target_property(blosc2libname Blosc2::blosc2_static OUTPUT_NAME) # fallback because it fails for me but I think it may be a bug. _error_if_not_found("OUTPUT_NAME" ${blosc2libname} "blosc2") - set(BLOSC2_LIB_NAME ${blosc2libname}) + set(BLOSC2_LIB_NAME $, "${blosc2libname}${CMAKE_DEBUG_POSTFIX}", "${blosc2libname}">) endif() endif() From 05d9457b3b9a372ab101c581e450e2ebc26d0bea Mon Sep 17 00:00:00 2001 From: Philippe Leprince Date: Tue, 2 Apr 2024 15:33:02 +0200 Subject: [PATCH 14/34] fix: debug build was not linking against debug blosc lib, for older cmake. Signed-off-by: Philippe Leprince --- cmake/OpenEXRSetup.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/OpenEXRSetup.cmake b/cmake/OpenEXRSetup.cmake index 064ec8e4bc..fa598ff621 100644 --- a/cmake/OpenEXRSetup.cmake +++ b/cmake/OpenEXRSetup.cmake @@ -401,7 +401,7 @@ if(NOT TARGET Blosc2::blosc2_static AND NOT Blosc2_FOUND) set(BLOSC2_LIB_DIR ${blosc2libdir}) get_target_property(blosc2libname Blosc2::blosc2_static OUTPUT_NAME) - set(BLOSC2_LIB_NAME $, "${blosc2libname}${CMAKE_DEBUG_POSTFIX}", "${blosc2libname}">) + set(BLOSC2_LIB_NAME ${blosc2libname}$<$:${CMAKE_DEBUG_POSTFIX}>) endif() else() message(STATUS "Blosc2: Using installed Blosc2 ${Blosc2_VERSION} from ${Blosc2_DIR}") @@ -420,7 +420,7 @@ else() get_target_property(blosc2libname Blosc2::blosc2_static OUTPUT_NAME) # fallback because it fails for me but I think it may be a bug. _error_if_not_found("OUTPUT_NAME" ${blosc2libname} "blosc2") - set(BLOSC2_LIB_NAME $, "${blosc2libname}${CMAKE_DEBUG_POSTFIX}", "${blosc2libname}">) + set(BLOSC2_LIB_NAME ${blosc2libname}$<$:${CMAKE_DEBUG_POSTFIX}>) endif() endif() From 9799bf9d5c5132fdeea94678fd061ae3ba51025e Mon Sep 17 00:00:00 2001 From: Philippe Leprince Date: Tue, 2 Apr 2024 16:15:36 +0200 Subject: [PATCH 15/34] fix: yet another variation on the same theme Signed-off-by: Philippe Leprince --- cmake/LibraryDefine.cmake | 2 +- cmake/OpenEXRSetup.cmake | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/LibraryDefine.cmake b/cmake/LibraryDefine.cmake index 541d48aefd..17b1a403d4 100644 --- a/cmake/LibraryDefine.cmake +++ b/cmake/LibraryDefine.cmake @@ -38,7 +38,7 @@ function(OPENEXR_DEFINE_LIBRARY libname) message(STATUS ">> BLOSC2_LIB_NAME: ${BLOSC2_LIB_NAME}") target_include_directories(${objlib} PRIVATE ${BLOSC2_INCLUDE_DIRS}) target_link_directories(${objlib} PRIVATE ${BLOSC2_LIB_DIR}) - target_link_libraries(${objlib} PRIVATE "dl" ${BLOSC2_LIB_NAME}) + target_link_libraries(${objlib} PRIVATE "dl" ${BLOSC2_LIB_NAME}$<$:${CMAKE_DEBUG_POSTFIX}>) if(TARGET blosc2_static) install(TARGETS blosc2_static EXPORT ${objlib}) endif() diff --git a/cmake/OpenEXRSetup.cmake b/cmake/OpenEXRSetup.cmake index fa598ff621..48335a582a 100644 --- a/cmake/OpenEXRSetup.cmake +++ b/cmake/OpenEXRSetup.cmake @@ -401,7 +401,7 @@ if(NOT TARGET Blosc2::blosc2_static AND NOT Blosc2_FOUND) set(BLOSC2_LIB_DIR ${blosc2libdir}) get_target_property(blosc2libname Blosc2::blosc2_static OUTPUT_NAME) - set(BLOSC2_LIB_NAME ${blosc2libname}$<$:${CMAKE_DEBUG_POSTFIX}>) + set(BLOSC2_LIB_NAME ${blosc2libname}) endif() else() message(STATUS "Blosc2: Using installed Blosc2 ${Blosc2_VERSION} from ${Blosc2_DIR}") @@ -420,7 +420,7 @@ else() get_target_property(blosc2libname Blosc2::blosc2_static OUTPUT_NAME) # fallback because it fails for me but I think it may be a bug. _error_if_not_found("OUTPUT_NAME" ${blosc2libname} "blosc2") - set(BLOSC2_LIB_NAME ${blosc2libname}$<$:${CMAKE_DEBUG_POSTFIX}>) + set(BLOSC2_LIB_NAME ${blosc2libname}) endif() endif() From a06daecb6a906887283bb3fae0ccf2ab0326fa50 Mon Sep 17 00:00:00 2001 From: Philippe Leprince Date: Wed, 3 Apr 2024 17:09:13 +0200 Subject: [PATCH 16/34] fix: debug and static builds should work. Signed-off-by: Philippe Leprince --- cmake/LibraryDefine.cmake | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cmake/LibraryDefine.cmake b/cmake/LibraryDefine.cmake index 17b1a403d4..241cc1ea0f 100644 --- a/cmake/LibraryDefine.cmake +++ b/cmake/LibraryDefine.cmake @@ -38,7 +38,11 @@ function(OPENEXR_DEFINE_LIBRARY libname) message(STATUS ">> BLOSC2_LIB_NAME: ${BLOSC2_LIB_NAME}") target_include_directories(${objlib} PRIVATE ${BLOSC2_INCLUDE_DIRS}) target_link_directories(${objlib} PRIVATE ${BLOSC2_LIB_DIR}) - target_link_libraries(${objlib} PRIVATE "dl" ${BLOSC2_LIB_NAME}$<$:${CMAKE_DEBUG_POSTFIX}>) + if(BUILD_SHARED_LIBS) + target_link_libraries(${objlib} PRIVATE "dl" ${BLOSC2_LIB_NAME}$<$:${CMAKE_DEBUG_POSTFIX}>) + else() + target_link_libraries(${objlib} PUBLIC Blosc2::blosc2_static) + endif() if(TARGET blosc2_static) install(TARGETS blosc2_static EXPORT ${objlib}) endif() From 2d678aefc30a68b007a91c4c0f2d0b4605ecf1ae Mon Sep 17 00:00:00 2001 From: Philippe Leprince Date: Wed, 3 Apr 2024 17:23:58 +0200 Subject: [PATCH 17/34] fix: update bazel build Signed-off-by: Philippe Leprince --- BUILD.bazel | 2 ++ MODULE.bazel | 1 + 2 files changed, 3 insertions(+) diff --git a/BUILD.bazel b/BUILD.bazel index e233492d4b..660ecee6f3 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -259,6 +259,7 @@ cc_library( deps = [ "@imath//:Imath", "@libdeflate//:deflate", + "@c-blosc2//c-blosc2", ], ) @@ -502,6 +503,7 @@ cc_library( ":IlmThread", ":OpenEXRCore", "@imath//:Imath", + "@c-blosc2//c-blosc2", ], ) diff --git a/MODULE.bazel b/MODULE.bazel index 8742e50fd6..65763e87c1 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -10,3 +10,4 @@ bazel_dep(name = "bazel_skylib", version = "1.5.0") bazel_dep(name = "imath", version = "3.1.11") bazel_dep(name = "libdeflate", version = "1.19") bazel_dep(name = "platforms", version = "0.0.8") +bazel_dep(name = "c-blosc2", version = "2.12.0") From 2ad98fe683ed7b65de824e0d02e6f811d8d9ec00 Mon Sep 17 00:00:00 2001 From: Philippe Leprince Date: Wed, 3 Apr 2024 18:24:16 +0200 Subject: [PATCH 18/34] fix: potential bazel typo Signed-off-by: Philippe Leprince --- BUILD.bazel | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/BUILD.bazel b/BUILD.bazel index 660ecee6f3..5896187ab4 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -259,7 +259,7 @@ cc_library( deps = [ "@imath//:Imath", "@libdeflate//:deflate", - "@c-blosc2//c-blosc2", + "@c-blosc2//:c-blosc2", ], ) @@ -503,7 +503,7 @@ cc_library( ":IlmThread", ":OpenEXRCore", "@imath//:Imath", - "@c-blosc2//c-blosc2", + "@c-blosc2//:c-blosc2", ], ) From f2bdfdea6003cbdff4539cc68841baa159b4e3fe Mon Sep 17 00:00:00 2001 From: Philippe Leprince Date: Thu, 4 Apr 2024 19:18:41 +0200 Subject: [PATCH 19/34] fix: refactored linking and fixed validation Signed-off-by: Philippe Leprince --- cmake/CMakeLists.txt | 1 + cmake/LibraryDefine.cmake | 11 +++-------- cmake/OpenEXR.pc.in | 3 ++- cmake/OpenEXRSetup.cmake | 8 -------- src/lib/OpenEXR/CMakeLists.txt | 1 + 5 files changed, 7 insertions(+), 17 deletions(-) diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 9348702611..3e410f7134 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -192,6 +192,7 @@ if(OPENEXR_INSTALL_PKG_CONFIG) function(openexr_pkg_config_help pcinfile) string(TOUPPER "${CMAKE_BUILD_TYPE}" uppercase_CMAKE_BUILD_TYPE) set(LIB_SUFFIX_DASH ${OPENEXR_LIB_SUFFIX}${CMAKE_${uppercase_CMAKE_BUILD_TYPE}_POSTFIX}) + set(LIB_BUILD_SUFFIX ${CMAKE_${uppercase_CMAKE_BUILD_TYPE}_POSTFIX}) if(OPENEXR_ENABLE_THREADING AND TARGET Threads::Threads) # hrm, can't use properties as they end up as generator expressions # which don't seem to evaluate diff --git a/cmake/LibraryDefine.cmake b/cmake/LibraryDefine.cmake index 241cc1ea0f..ce1a21362f 100644 --- a/cmake/LibraryDefine.cmake +++ b/cmake/LibraryDefine.cmake @@ -35,19 +35,14 @@ function(OPENEXR_DEFINE_LIBRARY libname) message(STATUS "Blosc2: setting up for ${objlib}...") message(STATUS ">> BLOSC2_INCLUDE_DIRS: ${BLOSC2_INCLUDE_DIRS}") message(STATUS ">> BLOSC2_LIB_DIR: ${BLOSC2_LIB_DIR}") - message(STATUS ">> BLOSC2_LIB_NAME: ${BLOSC2_LIB_NAME}") target_include_directories(${objlib} PRIVATE ${BLOSC2_INCLUDE_DIRS}) target_link_directories(${objlib} PRIVATE ${BLOSC2_LIB_DIR}) - if(BUILD_SHARED_LIBS) - target_link_libraries(${objlib} PRIVATE "dl" ${BLOSC2_LIB_NAME}$<$:${CMAKE_DEBUG_POSTFIX}>) - else() - target_link_libraries(${objlib} PUBLIC Blosc2::blosc2_static) - endif() - if(TARGET blosc2_static) + target_link_libraries(${objlib} PRIVATE Blosc2::blosc2_static "dl") + # install the static library if not using the installed lib. + if(TARGET blosc2_static AND NOT Blosc2_FOUND) install(TARGETS blosc2_static EXPORT ${objlib}) endif() endif() - if(OPENEXR_CURLIB_PRIV_EXPORT AND BUILD_SHARED_LIBS) target_compile_definitions(${objlib} PRIVATE ${OPENEXR_CURLIB_PRIV_EXPORT}) diff --git a/cmake/OpenEXR.pc.in b/cmake/OpenEXR.pc.in index 68d71c3c84..cd9e4cc8b5 100644 --- a/cmake/OpenEXR.pc.in +++ b/cmake/OpenEXR.pc.in @@ -9,12 +9,13 @@ libdir=@PKG_CONFIG_INSTALL_LIBDIR@ includedir=@PKG_CONFIG_INSTALL_INCLUDEDIR@ OpenEXR_includedir=${includedir}/OpenEXR libsuffix=@LIB_SUFFIX_DASH@ +libbuildsuffix=@LIB_BUILD_SUFFIX@ Name: OpenEXR Description: OpenEXR image library Version: @OPENEXR_VERSION@ -Libs: @exr_pthread_libs@ -L${libdir} -lOpenEXR${libsuffix} -lOpenEXRUtil${libsuffix} -lOpenEXRCore${libsuffix} -lIex${libsuffix} -lIlmThread${libsuffix} +Libs: @exr_pthread_libs@ -L${libdir} -lOpenEXR${libsuffix} -lOpenEXRUtil${libsuffix} -lOpenEXRCore${libsuffix} -lIex${libsuffix} -lIlmThread${libsuffix} -lblosc2${libbuildsuffix} -ldl Cflags: -I${includedir} -I${OpenEXR_includedir} @exr_pthread_cflags@ Requires: Imath Requires.private: @EXR_DEFLATE_PKGCONFIG_REQUIRES@ diff --git a/cmake/OpenEXRSetup.cmake b/cmake/OpenEXRSetup.cmake index 48335a582a..8e1a8b124f 100644 --- a/cmake/OpenEXRSetup.cmake +++ b/cmake/OpenEXRSetup.cmake @@ -399,9 +399,6 @@ if(NOT TARGET Blosc2::blosc2_static AND NOT Blosc2_FOUND) get_target_property(blosc2libdir Blosc2::blosc2_static BINARY_DIR) set(BLOSC2_LIB_DIR ${blosc2libdir}) - - get_target_property(blosc2libname Blosc2::blosc2_static OUTPUT_NAME) - set(BLOSC2_LIB_NAME ${blosc2libname}) endif() else() message(STATUS "Blosc2: Using installed Blosc2 ${Blosc2_VERSION} from ${Blosc2_DIR}") @@ -416,11 +413,6 @@ else() get_target_property(blosc2libdir Blosc2::blosc2_static BINARY_DIR) _error_if_not_found("BINARY_DIR" ${blosc2libdir} "") set(BLOSC2_LIB_DIR ${blosc2libdir}) - - get_target_property(blosc2libname Blosc2::blosc2_static OUTPUT_NAME) - # fallback because it fails for me but I think it may be a bug. - _error_if_not_found("OUTPUT_NAME" ${blosc2libname} "blosc2") - set(BLOSC2_LIB_NAME ${blosc2libname}) endif() endif() diff --git a/src/lib/OpenEXR/CMakeLists.txt b/src/lib/OpenEXR/CMakeLists.txt index cc7d7ff1c9..2ce8729e57 100644 --- a/src/lib/OpenEXR/CMakeLists.txt +++ b/src/lib/OpenEXR/CMakeLists.txt @@ -219,6 +219,7 @@ openexr_define_library(OpenEXR ImfXdr.h DEPENDENCIES Imath::Imath + Blosc2::blosc2_static OpenEXR::Config OpenEXR::Iex OpenEXR::IlmThread From a887716cf281a0a83d408e4980841c4558fd9587 Mon Sep 17 00:00:00 2001 From: Philippe Leprince Date: Thu, 4 Apr 2024 21:07:25 +0200 Subject: [PATCH 20/34] fix: update bazel blosc version Signed-off-by: Philippe Leprince --- MODULE.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MODULE.bazel b/MODULE.bazel index 65763e87c1..63d3f067de 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -10,4 +10,4 @@ bazel_dep(name = "bazel_skylib", version = "1.5.0") bazel_dep(name = "imath", version = "3.1.11") bazel_dep(name = "libdeflate", version = "1.19") bazel_dep(name = "platforms", version = "0.0.8") -bazel_dep(name = "c-blosc2", version = "2.12.0") +bazel_dep(name = "c-blosc2", version = "2.12.0.bcr.1") From 688d0d66634bfd92003ab3d30cecc85007314d5e Mon Sep 17 00:00:00 2001 From: Philippe Leprince Date: Fri, 5 Apr 2024 10:38:48 +0200 Subject: [PATCH 21/34] fix: tentative: always enable threads to build examples as it is needed for blosc2. Signed-off-by: Philippe Leprince --- cmake/OpenEXRConfig.cmake.in | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/cmake/OpenEXRConfig.cmake.in b/cmake/OpenEXRConfig.cmake.in index e94fd7b275..288fde2217 100644 --- a/cmake/OpenEXRConfig.cmake.in +++ b/cmake/OpenEXRConfig.cmake.in @@ -5,12 +5,10 @@ include(CMakeFindDependencyMacro) -set(openexr_needthreads @OPENEXR_ENABLE_THREADING@) -if (openexr_needthreads) - set(THREADS_PREFER_PTHREAD_FLAG ON) - find_dependency(Threads) -endif() -unset(openexr_needthreads) +# blosc2 needs threads, so we set it irrespective of OPENEXR_ENABLE_THREADING +# which enables threaded processing of requests. +set(THREADS_PREFER_PTHREAD_FLAG ON) +find_dependency(Threads) find_dependency(Imath) From 387d2ca59f77a1ea37b44e227ffb058e9a52d039 Mon Sep 17 00:00:00 2001 From: Philippe Leprince Date: Fri, 5 Apr 2024 11:37:06 +0200 Subject: [PATCH 22/34] fix: Replacing "dl" with CMAKE_DL_LIBS to fix windows build Signed-off-by: Philippe Leprince --- cmake/LibraryDefine.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/LibraryDefine.cmake b/cmake/LibraryDefine.cmake index ce1a21362f..1b788efd16 100644 --- a/cmake/LibraryDefine.cmake +++ b/cmake/LibraryDefine.cmake @@ -37,7 +37,7 @@ function(OPENEXR_DEFINE_LIBRARY libname) message(STATUS ">> BLOSC2_LIB_DIR: ${BLOSC2_LIB_DIR}") target_include_directories(${objlib} PRIVATE ${BLOSC2_INCLUDE_DIRS}) target_link_directories(${objlib} PRIVATE ${BLOSC2_LIB_DIR}) - target_link_libraries(${objlib} PRIVATE Blosc2::blosc2_static "dl") + target_link_libraries(${objlib} PRIVATE Blosc2::blosc2_static ${CMAKE_DL_LIBS}) # install the static library if not using the installed lib. if(TARGET blosc2_static AND NOT Blosc2_FOUND) install(TARGETS blosc2_static EXPORT ${objlib}) From 06f30a848cd2e6dab7a846ed6e4992959fea40fc Mon Sep 17 00:00:00 2001 From: Philippe Leprince Date: Fri, 5 Apr 2024 12:03:30 +0200 Subject: [PATCH 23/34] fix: update linking for CI fuzzing Signed-off-by: Philippe Leprince --- src/test/OpenEXRFuzzTest/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/OpenEXRFuzzTest/CMakeLists.txt b/src/test/OpenEXRFuzzTest/CMakeLists.txt index 4a2c0b3594..7bc867f710 100644 --- a/src/test/OpenEXRFuzzTest/CMakeLists.txt +++ b/src/test/OpenEXRFuzzTest/CMakeLists.txt @@ -17,7 +17,7 @@ if(OPENEXR_RUN_FUZZ_TESTS) testFuzzTiles.h ) target_include_directories(OpenEXRFuzzTest PRIVATE ../OpenEXRTest) - target_link_libraries(OpenEXRFuzzTest OpenEXR::OpenEXR) + target_link_libraries(OpenEXRFuzzTest OpenEXR::OpenEXR blosc2 ${CMAKE_DL_LIBS}) set_target_properties(OpenEXRFuzzTest PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin" ) From 903c2c4b00d3e3b7a453019beb46a58ad3192fa4 Mon Sep 17 00:00:00 2001 From: Philippe Leprince Date: Fri, 5 Apr 2024 13:32:35 +0200 Subject: [PATCH 24/34] fix(bazel): add missing source files to BUILD.bazel Signed-off-by: Philippe Leprince --- BUILD.bazel | 3 +++ 1 file changed, 3 insertions(+) diff --git a/BUILD.bazel b/BUILD.bazel index 5896187ab4..eff97d0cc8 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -202,6 +202,7 @@ cc_library( "src/lib/OpenEXRCore/internal_win32_file_impl.h", "src/lib/OpenEXRCore/internal_xdr.h", "src/lib/OpenEXRCore/internal_zip.c", + "src/lib/OpenEXRCore/internal_zstd.c", "src/lib/OpenEXRCore/memory.c", "src/lib/OpenEXRCore/opaque.c", "src/lib/OpenEXRCore/openexr_version.h", @@ -358,6 +359,7 @@ cc_library( "src/lib/OpenEXR/ImfWav.cpp", "src/lib/OpenEXR/ImfZip.cpp", "src/lib/OpenEXR/ImfZipCompressor.cpp", + "src/lib/OpenEXR/ImfZstdCompressor.cpp", "src/lib/OpenEXR/b44ExpLogTable.h", "src/lib/OpenEXR/dwaLookups.h", ], @@ -477,6 +479,7 @@ cc_library( "src/lib/OpenEXR/ImfXdr.h", "src/lib/OpenEXR/ImfZip.h", "src/lib/OpenEXR/ImfZipCompressor.h", + "src/lib/OpenEXR/ImfZstdCompressor.h", "src/lib/OpenEXR/OpenEXRConfig.h", "src/lib/OpenEXR/OpenEXRConfigInternal.h", ], From 40b1937993f1dbb413bb85fdd356cc295a56ad4f Mon Sep 17 00:00:00 2001 From: Philippe Leprince Date: Fri, 5 Apr 2024 13:40:39 +0200 Subject: [PATCH 25/34] fix(bazel): tentative: add blosc3 deps to example build Signed-off-by: Philippe Leprince --- src/examples/BUILD.bazel | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/examples/BUILD.bazel b/src/examples/BUILD.bazel index 4db5623735..2763558305 100644 --- a/src/examples/BUILD.bazel +++ b/src/examples/BUILD.bazel @@ -24,5 +24,8 @@ cc_test( "rgbaInterfaceTiledExamples.cpp", "rgbaInterfaceTiledExamples.h", ], - deps = ["//:OpenEXR"], + deps = [ + "//:OpenEXR", + "@c-blosc2//:c-blosc2", + ], ) From bb8754ad8ee4fed0280b007f57c01adb0090897b Mon Sep 17 00:00:00 2001 From: Philippe Leprince Date: Fri, 5 Apr 2024 13:47:23 +0200 Subject: [PATCH 26/34] fix(bazel): tentative: add linkopts to example build Signed-off-by: Philippe Leprince --- src/examples/BUILD.bazel | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/examples/BUILD.bazel b/src/examples/BUILD.bazel index 2763558305..dad4c099a2 100644 --- a/src/examples/BUILD.bazel +++ b/src/examples/BUILD.bazel @@ -26,6 +26,9 @@ cc_test( ], deps = [ "//:OpenEXR", - "@c-blosc2//:c-blosc2", ], + linkopts = [ + "-lblosc2", + "-ldl" + ] ) From 15844b5fcb3f9af5f5b01d254e83405dc1a63907 Mon Sep 17 00:00:00 2001 From: Philippe Leprince Date: Fri, 5 Apr 2024 13:50:20 +0200 Subject: [PATCH 27/34] fix(bazel): tentative: add linkopts+deps to example build Signed-off-by: Philippe Leprince --- src/examples/BUILD.bazel | 1 + 1 file changed, 1 insertion(+) diff --git a/src/examples/BUILD.bazel b/src/examples/BUILD.bazel index dad4c099a2..0c9a6acda4 100644 --- a/src/examples/BUILD.bazel +++ b/src/examples/BUILD.bazel @@ -26,6 +26,7 @@ cc_test( ], deps = [ "//:OpenEXR", + "@c-blosc2//:c-blosc2", ], linkopts = [ "-lblosc2", From a9ebf07132dda49c14b376b7737088ec5b2bb2d6 Mon Sep 17 00:00:00 2001 From: Philippe Leprince Date: Fri, 5 Apr 2024 15:53:07 +0200 Subject: [PATCH 28/34] fix(bazel): everything but the examples build. Can't make it work locally. Signed-off-by: Philippe Leprince --- BUILD.bazel | 6 ++++-- src/examples/BUILD.bazel | 5 ----- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/BUILD.bazel b/BUILD.bazel index eff97d0cc8..1b8b836daa 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -254,13 +254,14 @@ cc_library( ":windows": [], "//conditions:default": [ "-pthread", + "-ldl", ], }), visibility = ["//visibility:public"], deps = [ + "@c-blosc2//:c-blosc2", "@imath//:Imath", "@libdeflate//:deflate", - "@c-blosc2//:c-blosc2", ], ) @@ -499,14 +500,15 @@ cc_library( ":windows": [], "//conditions:default": [ "-pthread", + "-ldl", ], }), visibility = ["//visibility:public"], deps = [ + "@c-blosc2//:c-blosc2", ":IlmThread", ":OpenEXRCore", "@imath//:Imath", - "@c-blosc2//:c-blosc2", ], ) diff --git a/src/examples/BUILD.bazel b/src/examples/BUILD.bazel index 0c9a6acda4..d0229f1ea4 100644 --- a/src/examples/BUILD.bazel +++ b/src/examples/BUILD.bazel @@ -26,10 +26,5 @@ cc_test( ], deps = [ "//:OpenEXR", - "@c-blosc2//:c-blosc2", ], - linkopts = [ - "-lblosc2", - "-ldl" - ] ) From 55f37b0017d22462a434c5399c1aeb18fb5c0b5d Mon Sep 17 00:00:00 2001 From: Philippe Leprince Date: Fri, 5 Apr 2024 16:05:44 +0200 Subject: [PATCH 29/34] fix(fuzz): tentative: use target name to link Signed-off-by: Philippe Leprince --- src/test/OpenEXRFuzzTest/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/OpenEXRFuzzTest/CMakeLists.txt b/src/test/OpenEXRFuzzTest/CMakeLists.txt index 7bc867f710..e787b8a042 100644 --- a/src/test/OpenEXRFuzzTest/CMakeLists.txt +++ b/src/test/OpenEXRFuzzTest/CMakeLists.txt @@ -17,7 +17,7 @@ if(OPENEXR_RUN_FUZZ_TESTS) testFuzzTiles.h ) target_include_directories(OpenEXRFuzzTest PRIVATE ../OpenEXRTest) - target_link_libraries(OpenEXRFuzzTest OpenEXR::OpenEXR blosc2 ${CMAKE_DL_LIBS}) + target_link_libraries(OpenEXRFuzzTest OpenEXR::OpenEXR blosc2_static ${CMAKE_DL_LIBS}) set_target_properties(OpenEXRFuzzTest PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin" ) From 4c714a2f30e2015aa362c6db31ad0911b74e018c Mon Sep 17 00:00:00 2001 From: Philippe Leprince Date: Tue, 23 Apr 2024 15:28:22 +0200 Subject: [PATCH 30/34] build(bazel): switch c-blosc2 tp 2.12.0.bcr.2 Signed-off-by: Philippe Leprince --- MODULE.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MODULE.bazel b/MODULE.bazel index 63d3f067de..ba4e4e2d8f 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -10,4 +10,4 @@ bazel_dep(name = "bazel_skylib", version = "1.5.0") bazel_dep(name = "imath", version = "3.1.11") bazel_dep(name = "libdeflate", version = "1.19") bazel_dep(name = "platforms", version = "0.0.8") -bazel_dep(name = "c-blosc2", version = "2.12.0.bcr.1") +bazel_dep(name = "c-blosc2", version = "2.12.0.bcr.2") From 81e17011bcae03cd037329c7876ed467975075b2 Mon Sep 17 00:00:00 2001 From: Philippe Leprince Date: Tue, 23 Apr 2024 16:19:14 +0200 Subject: [PATCH 31/34] build(fuzz): build fuzz with dwarf-4. (test) Signed-off-by: Philippe Leprince --- src/test/OpenEXRFuzzTest/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/OpenEXRFuzzTest/CMakeLists.txt b/src/test/OpenEXRFuzzTest/CMakeLists.txt index e787b8a042..a46af33bda 100644 --- a/src/test/OpenEXRFuzzTest/CMakeLists.txt +++ b/src/test/OpenEXRFuzzTest/CMakeLists.txt @@ -17,6 +17,7 @@ if(OPENEXR_RUN_FUZZ_TESTS) testFuzzTiles.h ) target_include_directories(OpenEXRFuzzTest PRIVATE ../OpenEXRTest) + target_compile_options(OpenEXRFuzzTest PRIVATE "-gdwarf-4") target_link_libraries(OpenEXRFuzzTest OpenEXR::OpenEXR blosc2_static ${CMAKE_DL_LIBS}) set_target_properties(OpenEXRFuzzTest PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin" From 5b0be1dc4b075d27f53146e3d736dabb47174395 Mon Sep 17 00:00:00 2001 From: Philippe Leprince Date: Tue, 23 Apr 2024 16:51:56 +0200 Subject: [PATCH 32/34] build(fuzz): build fuzz with dwarf-4. (test #2) Signed-off-by: Philippe Leprince --- src/test/OpenEXRFuzzTest/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/OpenEXRFuzzTest/CMakeLists.txt b/src/test/OpenEXRFuzzTest/CMakeLists.txt index a46af33bda..3b597d5acc 100644 --- a/src/test/OpenEXRFuzzTest/CMakeLists.txt +++ b/src/test/OpenEXRFuzzTest/CMakeLists.txt @@ -17,7 +17,7 @@ if(OPENEXR_RUN_FUZZ_TESTS) testFuzzTiles.h ) target_include_directories(OpenEXRFuzzTest PRIVATE ../OpenEXRTest) - target_compile_options(OpenEXRFuzzTest PRIVATE "-gdwarf-4") + target_compile_options(OpenEXRFuzzTest PUBLIC "-gdwarf-4") target_link_libraries(OpenEXRFuzzTest OpenEXR::OpenEXR blosc2_static ${CMAKE_DL_LIBS}) set_target_properties(OpenEXRFuzzTest PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin" From cb6a1e07dfadda8322aae81712e45c4022dd4a4b Mon Sep 17 00:00:00 2001 From: Philippe Leprince Date: Tue, 23 Apr 2024 17:12:58 +0200 Subject: [PATCH 33/34] build(fuzz): build fuzz with dwarf-4. (test #3) Signed-off-by: Philippe Leprince --- cmake/OpenEXRSetup.cmake | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmake/OpenEXRSetup.cmake b/cmake/OpenEXRSetup.cmake index 8e1a8b124f..9aaf7bad9d 100644 --- a/cmake/OpenEXRSetup.cmake +++ b/cmake/OpenEXRSetup.cmake @@ -399,6 +399,10 @@ if(NOT TARGET Blosc2::blosc2_static AND NOT Blosc2_FOUND) get_target_property(blosc2libdir Blosc2::blosc2_static BINARY_DIR) set(BLOSC2_LIB_DIR ${blosc2libdir}) + + if(OPENEXR_RUN_FUZZ_TESTS) + target_compile_options(blosc2_static PUBLIC "-gdwarf-4") + endif() endif() else() message(STATUS "Blosc2: Using installed Blosc2 ${Blosc2_VERSION} from ${Blosc2_DIR}") From 5047f7f8bbb55e2477be898d7155ec13ec7e04a4 Mon Sep 17 00:00:00 2001 From: Philippe Leprince Date: Wed, 19 Jun 2024 11:56:34 +0200 Subject: [PATCH 34/34] ci: temporarily disable fuzz test This code will need to be removed before merging. Signed-off-by: Philippe Leprince --- .github/workflows/ossfuzz_workflow.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ossfuzz_workflow.yml b/.github/workflows/ossfuzz_workflow.yml index 390cfe43f9..86fc78bda4 100644 --- a/.github/workflows/ossfuzz_workflow.yml +++ b/.github/workflows/ossfuzz_workflow.yml @@ -25,6 +25,12 @@ on: - '!src/wrappers/**' - '!.github/workflows/**' - '.github/workflows/ossfuzz_workflow.yml' + # FIXME: Fuzzing build fails because we need to add libblosc2 to the list + # of linked libs in: + # https://github.com/google/oss-fuzz/blob/master/projects/openexr/build.sh + # ... but we can only do it once this PR is merged. + branches-ignore: # FIXME: REMOVE THIS CONDITION BEFORE MERGING ! + - 'main' permissions: contents: read