Add User APIs to Read and Write DCZ Header Frames

Author: felixhandte

lib/common/zstd_internal.h

@@ -32,6 +32,7 @@
 #ifndef XXH_STATIC_LINKING_ONLY
 #  define XXH_STATIC_LINKING_ONLY  /* XXH64_state_t */
 #endif
+#include "sha256.h"                /* ZSTD_SHA256_DIGEST_SIZE */
 #include "xxhash.h"                /* XXH_reset, update, digest */
 #ifndef ZSTD_NO_TRACE
 #  include "zstd_trace.h"
@@ -87,6 +88,9 @@ typedef enum { bt_raw, bt_rle, bt_compressed, bt_reserved } blockType_e;
 
 #define ZSTD_FRAMECHECKSUMSIZE 4
 
+#define ZSTD_HTTPDCZ_HEADER_SIZE (ZSTD_SKIPPABLEHEADERSIZE + ZSTD_SHA256_DIGEST_SIZE)
+#define ZSTD_HTTPDCZ_HEADER_SKIPPABLE_VARIANT (0x0e)
+
 #define MIN_SEQUENCES_SIZE 1 /* nbSeq==0 */
 #define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */)   /* for a non-null block */
 #define MIN_LITERALS_FOR_4_STREAMS 6

lib/compress/zstd_compress.c

@@ -28,7 +28,8 @@
 #include "zstd_opt.h"
 #include "zstd_ldm.h"
 #include "zstd_compress_superblock.h"
-#include  "../common/bits.h"      /* ZSTD_highbit32, ZSTD_rotateRight_U64 */
+#include "../common/bits.h"    /* ZSTD_highbit32, ZSTD_rotateRight_U64 */
+#include "../common/sha256.h"  /* ZSTD_SHA256_Result, ZSTD_SHA256_hash, ZSTD_SHA256_DIGEST_SIZE */
 
 /* ***************************************************************
 *  Tuning parameters
@@ -5200,7 +5201,7 @@ ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs,
                                void* workspace)
 {
     DEBUGLOG(4, "ZSTD_compress_insertDictionary (dictSize=%u)", (U32)dictSize);
-    if ((dict==NULL) || (dictSize<8)) {
+    if ((dict == NULL) || (dictSize < ZSTD_DICTIONARYSIZE_MIN)) {
         RETURN_ERROR_IF(dictContentType == ZSTD_dct_fullDict, dictionary_wrong, "");
         return 0;
     }
@@ -7836,3 +7837,24 @@ void ZSTD_CCtxParams_registerSequenceProducer(
         params->extSeqProdState = NULL;
     }
 }
+
+size_t ZSTD_writeHeaderForHTTPDCZ(
+        void* dst, size_t dstCapacity,
+        const void* dict, size_t dictSize) {
+    ZSTD_SHA256_Result hash;
+    RETURN_ERROR_IF(dst == NULL, dstBuffer_null, "NULL dst buffer.");
+    RETURN_ERROR_IF(dstCapacity < ZSTD_HTTPDCZ_HEADER_SIZE, dstSize_tooSmall, "Too small to write frame header.");
+    RETURN_ERROR_IF(dict == NULL, dictionary_corrupted, "Dictionary invalid: NULL pointer.");
+    RETURN_ERROR_IF(dictSize < ZSTD_DICTIONARYSIZE_MIN, dictionary_corrupted, "Dictionary invalid: too small.");
+
+    hash = ZSTD_SHA256_hash(dict, dictSize);
+    return ZSTD_writeSkippableFrame(dst, dstCapacity, hash.digest, sizeof(hash.digest), ZSTD_HTTPDCZ_HEADER_SKIPPABLE_VARIANT);
+}
+
+size_t ZSTD_writeHeaderForHTTPDCZ_fromCDict(
+        void* dst, size_t dstCapacity,
+        const ZSTD_CDict* cdict) {
+    RETURN_ERROR_IF(cdict == NULL, parameter_unsupported, "NULL cdict pointer.");
+    RETURN_ERROR_IF(cdict->dictContentType != ZSTD_dct_rawContent, dictionary_corrupted, "Dictionary invalid: must be loaded as raw content for DCZ.");
+    return ZSTD_writeHeaderForHTTPDCZ(dst, dstCapacity, cdict->dictContent, cdict->dictContentSize);
+}

lib/decompress/zstd_ddict.c

@@ -55,6 +55,12 @@ size_t ZSTD_DDict_dictSize(const ZSTD_DDict* ddict)
     return ddict->dictSize;
 }
 
+ZSTD_dictContentType_e ZSTD_DDict_type(const ZSTD_DDict* ddict)
+{
+    assert(ddict != NULL);
+    return (ddict->entropyPresent) ? ZSTD_dct_fullDict : ZSTD_dct_rawContent;
+}
+
 void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict)
 {
     DEBUGLOG(4, "ZSTD_copyDDictParameters");

lib/decompress/zstd_ddict.h

@@ -37,6 +37,8 @@
 const void* ZSTD_DDict_dictContent(const ZSTD_DDict* ddict);
 size_t ZSTD_DDict_dictSize(const ZSTD_DDict* ddict);
 
+ZSTD_dictContentType_e ZSTD_DDict_type(const ZSTD_DDict* ddict);
+
 void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
 
 

lib/decompress/zstd_decompress.c

@@ -64,7 +64,8 @@
 #define FSE_STATIC_LINKING_ONLY
 #include "../common/fse.h"
 #include "../common/huf.h"
-#include "../common/xxhash.h" /* XXH64_reset, XXH64_update, XXH64_digest, XXH64 */
+#include "../common/sha256.h"  /* ZSTD_SHA256_Result, ZSTD_SHA256_hash, ZSTD_SHA256_DIGEST_SIZE */
+#include "../common/xxhash.h"  /* XXH64_reset, XXH64_update, XXH64_digest, XXH64 */
 #include "zstd_decompress_internal.h"   /* ZSTD_DCtx */
 #include "zstd_ddict.h"  /* ZSTD_DDictDictContent */
 #include "zstd_decompress_block.h"   /* ZSTD_decompressBlock_internal */
@@ -623,6 +624,7 @@ size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity,
 
         /* check input validity */
         RETURN_ERROR_IF(!ZSTD_isSkippableFrame(src, srcSize), frameParameter_unsupported, "");
+        FORWARD_IF_ERROR(skippableFrameSize, "");
         RETURN_ERROR_IF(skippableFrameSize < ZSTD_SKIPPABLEHEADERSIZE || skippableFrameSize > srcSize, srcSize_wrong, "");
         RETURN_ERROR_IF(skippableContentSize > dstCapacity, dstSize_tooSmall, "");
 
@@ -2408,3 +2410,40 @@ size_t ZSTD_decompressStream_simpleArgs (
         return cErr;
     }
 }
+
+size_t ZSTD_readHeaderForHTTPDCZ(
+        void* dst, size_t dstCapacity,
+        const void* src, size_t srcSize) {
+    unsigned variant;
+    size_t result;
+    RETURN_ERROR_IF(dst == NULL, dstBuffer_null, "NULL dst buffer.");
+    RETURN_ERROR_IF(dstCapacity < ZSTD_SHA256_DIGEST_SIZE, dstSize_tooSmall, "Too small");
+
+    result = ZSTD_readSkippableFrame(dst, dstCapacity, &variant, src, srcSize);
+
+    FORWARD_IF_ERROR(result, "Couldn't read skippable frame.");
+    RETURN_ERROR_IF(variant != ZSTD_HTTPDCZ_HEADER_SKIPPABLE_VARIANT, prefix_unknown, "Skippable frame magic is wrong for DCZ.");
+    RETURN_ERROR_IF(result != ZSTD_SHA256_DIGEST_SIZE, corruption_detected, "Wrong skippable frame size for DCZ.");
+    return result;
+}
+
+size_t ZSTD_readHeaderForHTTPDCZ_validateDictMatches(
+        const void* src, size_t srcSize,
+        const void* dict, size_t dictSize) {
+    ZSTD_SHA256_Result frameHash, dictHash;
+    size_t result = ZSTD_readHeaderForHTTPDCZ(frameHash.digest, ZSTD_SHA256_DIGEST_SIZE, src, srcSize);
+    FORWARD_IF_ERROR(result, "Couldn't read DCZ header.");
+    RETURN_ERROR_IF(dict == NULL, dictionary_corrupted, "Dictionary invalid: NULL pointer.");
+    RETURN_ERROR_IF(dictSize < ZSTD_DICTIONARYSIZE_MIN, dictionary_corrupted, "Dictionary invalid: too small.");
+    dictHash = ZSTD_SHA256_hash(dict, dictSize);
+    RETURN_ERROR_IF(memcmp(frameHash.digest, dictHash.digest, ZSTD_SHA256_DIGEST_SIZE), dictionary_wrong, "DCZ hashes don't match.");
+    return 1;
+}
+
+size_t ZSTD_readHeaderForHTTPDCZ_validateDDictMatches(
+        const void* src, size_t srcSize,
+        const ZSTD_DDict* ddict) {
+    RETURN_ERROR_IF(ddict == NULL, parameter_unsupported, "NULL ddict pointer.");
+    RETURN_ERROR_IF(ZSTD_DDict_type(ddict) != ZSTD_dct_rawContent, dictionary_corrupted, "Dictionary invalid: must be loaded as raw content for DCZ.");
+    return ZSTD_readHeaderForHTTPDCZ_validateDictMatches(src, srcSize, ZSTD_DDict_dictContent(ddict), ZSTD_DDict_dictSize(ddict));
+}

lib/zstd.h

@@ -1259,6 +1259,8 @@ extern "C" {
 #define ZSTD_FRAMEHEADERSIZE_MAX   18   /* can be useful for static allocation */
 #define ZSTD_SKIPPABLEHEADERSIZE    8
 
+#define ZSTD_DICTIONARYSIZE_MIN     8
+
 /* compression parameter bounds */
 #define ZSTD_WINDOWLOG_MAX_32    30
 #define ZSTD_WINDOWLOG_MAX_64    31
@@ -1746,6 +1748,73 @@ ZSTDLIB_STATIC_API size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity,
  */
 ZSTDLIB_STATIC_API unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size);
 
+/*! ZSTD_writeHeaderForHTTPDCZ() :
+ *
+ * The `dcz` HTTP Content-Encoding specifies that payloads using that encoding
+ * begin with a skippable frame containing a SHA-256 hash of the dictionary
+ * contents used to compress the subsequent Zstandard frame(s).
+ *
+ * This function produces that header.
+ *
+ * Note: You must ensure you use the same dictionary provided to this function
+ *       for the associated compressions (obviously!).
+ *
+ * @return : Number of bytes written (always 40) or a ZSTD error.
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_writeHeaderForHTTPDCZ(
+        void* dst, size_t dstCapacity,
+        const void* dict, size_t dictSize);
+
+/*! ZSTD_writeHeaderForHTTPDCZ_fromCDict() :
+ *
+ * The `dcz` HTTP Content-Encoding specifies that payloads using that encoding
+ * begin with a skippable frame containing a SHA-256 hash of the dictionary
+ * contents used to compress the subsequent Zstandard frame(s).
+ *
+ * This function produces that header.
+ *
+ * Note: You must ensure you use the same dictionary provided to this function
+ *       for the associated compressions (obviously!).
+ *
+ * Note: The output of this is a function solely of the dictionary content and
+ *       not the input. This function has to hash the dictionary contents.
+ *
+ * @return : Number of bytes written (always 40) or a ZSTD error.
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_writeHeaderForHTTPDCZ_fromCDict(
+        void* dst, size_t dstCapacity,
+        const ZSTD_CDict* cdict);
+
+/*! ZSTD_readHeaderForHTTPDCZ() :
+ *
+ * This function checks that the provided input begins with a skippable
+ * frame conforming to the `dcz` HTTP Content-Encoding specification. If it
+ * does, it extracts the contained hash into the provided @dst buffer.
+ * @dstCapacity must be at least 32.
+ *
+ * @return : If the hash was successfully read, the number of bytes written
+ *           into @dst (always 32). Otherwise, a ZSTD error.
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_readHeaderForHTTPDCZ(
+        void* dst, size_t dstCapacity,
+        const void* src, size_t srcSize);
+
+/*! ZSTD_readHeaderForHTTPDCZ_validate*Matches() :
+ *
+ * This function checks that the provided input (a) begins with a skippable
+ * frame conforming to the `dcz` HTTP Content-Encoding specification and (b)
+ * that the hash indicated by that frame matches the hash of the provided
+ * dictionary.
+ *
+ * @return : 1, if the header was successfully read and the hashes match.
+ *           Otherwise, a ZSTD error.
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_readHeaderForHTTPDCZ_validateDictMatches(
+        const void* src, size_t srcSize,
+        const void* dict, size_t dictSize);
+ZSTDLIB_STATIC_API size_t ZSTD_readHeaderForHTTPDCZ_validateDDictMatches(
+        const void* src, size_t srcSize,
+        const ZSTD_DDict* ddict);
 
 
 /***************************************