libclamav: cache EVP_MD handles to fix per-hash provider-fetch churn and a FIPS memory leak

On OpenSSL 3, the hashing helpers (cl_hash_init, cl_hash_data,
cl_hash_file_fd and their _ex variants) created a fresh OSSL_LIB_CTX and
called EVP_MD_fetch() on every hash, then tore both down. A freshly-allocated
library context has no method-store cache, so each fetch pays the full
provider-bootstrap + method-construction cost (on the order of hundreds of
microseconds), and this runs for md5/sha1/sha2-256 on essentially every
scanned object (whole-file and section hashes plus the SHA2-256 clean-cache
lookup in fmap_get_hash). The per-object hashing plumbing ends up costing far
more than the hashing itself, whether or not the host is in FIPS mode.

Fetch each FIPS-bypass digest once into a single shared, long-lived non-FIPS
OSSL_LIB_CTX and reuse it (a fetched EVP_MD is reference counted and safe to
share across threads; the cache is guarded by a mutex). The FIPS-bypass
semantics are unchanged -- the shared context is still created with
OSSL_LIB_CTX_new() and digests are still fetched with the "-fips" property, so
MD5/SHA1 signature hashing still works on FIPS hosts where it can. The
FIPS-compliant (non-bypass) path is intentionally not cached: it is still
fetched from the default library context on every call so it keeps honoring
the current default property query.

This also removes the per-hash OSSL_LIB_CTX_new(), which fixes a memory leak.
The previous code allocated the context before the fetch and did not free it
on the "md == NULL" return path, so on a host whose FIPS policy makes an
algorithm unavailable (e.g. MD5), every hashed object leaked an OSSL_LIB_CTX.
Because libclamav hashes MD5 for nearly every scanned file, a full scan grew
without bound until it was OOM-killed. With a single shared context there is
nothing to leak per hash; an unavailable algorithm now degrades to a clean,
bounded failure instead of exhausting memory.

Author: Sewci0

libclamav/crypto.c

@@ -68,6 +68,10 @@
 #include "str.h"
 #include "iowrap.h"
 
+#ifdef CL_THREAD_SAFE
+#include <pthread.h>
+#endif
+
 #if defined(_WIN32)
 char *strptime(const char *buf, const char *fmt, struct tm *tm);
 #endif
@@ -162,6 +166,95 @@ void cl_cleanup_crypto(void)
     return;
 }
 
+#if OPENSSL_VERSION_MAJOR >= 3
+/*
+ * Digest handle cache.
+ *
+ * On OpenSSL 3, EVP_MD_fetch() into a freshly-allocated OSSL_LIB_CTX is
+ * expensive: the empty context must bootstrap a provider and construct the
+ * method from scratch (on the order of hundreds of microseconds). ClamAV
+ * hashes md5/sha1/sha2-256 for essentially every scanned object, so fetching
+ * per hash would dominate scan time while doing no actual hashing.
+ *
+ * Fetch each FIPS-bypass digest once into a single shared non-FIPS library
+ * context and reuse it (a fetched EVP_MD is reference counted and safe to share
+ * across threads). cli_get_md() returns a reference the caller owns and must
+ * release with EVP_MD_free().
+ *
+ * Only the bypass digests are cached. The FIPS-compliant path is fetched from
+ * the default library context on every call, so it keeps honoring the current
+ * default property query -- e.g. if the embedding process enables FIPS at
+ * runtime, a non-bypass MD5/SHA1 request must start failing rather than return
+ * a stale non-FIPS handle. That default-context fetch is already inexpensive
+ * because OpenSSL caches provider methods per library context.
+ */
+#ifdef CL_THREAD_SAFE
+static pthread_mutex_t md_cache_mutex = PTHREAD_MUTEX_INITIALIZER;
+#endif
+
+/* Single shared non-FIPS library context backing all "-fips" (bypass) fetches. */
+static OSSL_LIB_CTX *nonfips_libctx = NULL;
+
+static struct {
+    const char *alg; /* string literal returned by to_openssl_alg(); stable */
+    EVP_MD *md;
+} md_cache[8];
+static size_t md_cache_len = 0;
+
+static EVP_MD *cli_get_md(const char *alg, int fips_bypass)
+{
+    const char *ossl_alg = to_openssl_alg(alg);
+    EVP_MD *result       = NULL;
+    size_t i;
+
+    if (NULL == ossl_alg) {
+        return NULL;
+    }
+
+    if (!fips_bypass) {
+        /* FIPS-compliant path: fetch from the default library context every
+         * time so the current default property query -- including any runtime
+         * FIPS change -- is honored. */
+        return EVP_MD_fetch(NULL, ossl_alg, NULL);
+    }
+
+#ifdef CL_THREAD_SAFE
+    pthread_mutex_lock(&md_cache_mutex);
+#endif
+
+    for (i = 0; i < md_cache_len; i++) {
+        if (0 == strcmp(md_cache[i].alg, ossl_alg)) {
+            result = md_cache[i].md;
+            goto done;
+        }
+    }
+
+    /* Miss: create the shared non-FIPS context on first use and fetch once. */
+    if (NULL == nonfips_libctx) {
+        nonfips_libctx = OSSL_LIB_CTX_new();
+        if (NULL == nonfips_libctx) {
+            cli_errmsg("cli_get_md: Failed to create OpenSSL library context\n");
+            goto done;
+        }
+    }
+    result = EVP_MD_fetch(nonfips_libctx, ossl_alg, "-fips");
+    if (NULL != result && md_cache_len < sizeof(md_cache) / sizeof(md_cache[0])) {
+        md_cache[md_cache_len].alg = ossl_alg;
+        md_cache[md_cache_len].md  = result; /* cache holds this reference */
+        md_cache_len++;
+    }
+
+done:
+    if (NULL != result) {
+        EVP_MD_up_ref(result); /* give the caller its own reference to free */
+    }
+#ifdef CL_THREAD_SAFE
+    pthread_mutex_unlock(&md_cache_mutex);
+#endif
+    return result;
+}
+#endif /* OPENSSL_VERSION_MAJOR >= 3 */
+
 /**
  * @brief Generate a hash of data.
  *
@@ -218,20 +311,7 @@ extern cl_error_t cl_hash_data_ex(
     }
 
 #if OPENSSL_VERSION_MAJOR >= 3
-    if (flags & CL_HASH_FLAG_FIPS_BYPASS) {
-        /* Bypass FIPS restrictions the OpenSSL 3.0 way */
-        ossl_ctx = OSSL_LIB_CTX_new();
-        if (NULL == ossl_ctx) {
-            cli_errmsg("cl_hash_data_ex: Failed to create new OpenSSL library context\n");
-            status = CL_EMEM;
-            goto done;
-        }
-
-        md = EVP_MD_fetch(ossl_ctx, to_openssl_alg(alg), "-fips");
-    } else {
-        /* Use FIPS compliant algorithms */
-        md = EVP_MD_fetch(NULL, to_openssl_alg(alg), NULL);
-    }
+    md = cli_get_md(alg, (flags & CL_HASH_FLAG_FIPS_BYPASS) ? 1 : 0);
 #else
     md = EVP_get_digestbyname(to_openssl_alg(alg));
 #endif
@@ -369,20 +449,7 @@ extern cl_error_t cl_hash_init_ex(
     }
 
 #if OPENSSL_VERSION_MAJOR >= 3
-    if (flags & CL_HASH_FLAG_FIPS_BYPASS) {
-        /* Bypass FIPS restrictions the OpenSSL 3.0 way */
-        ossl_ctx = OSSL_LIB_CTX_new();
-        if (NULL == ossl_ctx) {
-            cli_errmsg("cl_hash_data_ex: Failed to create new OpenSSL library context\n");
-            status = CL_EMEM;
-            goto done;
-        }
-
-        md = EVP_MD_fetch(ossl_ctx, to_openssl_alg(alg), "-fips");
-    } else {
-        /* Use FIPS compliant algorithms */
-        md = EVP_MD_fetch(NULL, to_openssl_alg(alg), NULL);
-    }
+    md = cli_get_md(alg, (flags & CL_HASH_FLAG_FIPS_BYPASS) ? 1 : 0);
 #else
     md = EVP_get_digestbyname(to_openssl_alg(alg));
 #endif
@@ -641,20 +708,7 @@ extern cl_error_t cl_hash_file_fd_ex(
     }
 
 #if OPENSSL_VERSION_MAJOR >= 3
-    if (flags & CL_HASH_FLAG_FIPS_BYPASS) {
-        /* Bypass FIPS restrictions the OpenSSL 3.0 way */
-        ossl_ctx = OSSL_LIB_CTX_new();
-        if (NULL == ossl_ctx) {
-            cli_errmsg("cl_hash_data_ex: Failed to create new OpenSSL library context\n");
-            status = CL_EMEM;
-            goto done;
-        }
-
-        md = EVP_MD_fetch(ossl_ctx, to_openssl_alg(alg), "-fips");
-    } else {
-        /* Use FIPS compliant algorithms */
-        md = EVP_MD_fetch(NULL, to_openssl_alg(alg), NULL);
-    }
+    md = cli_get_md(alg, (flags & CL_HASH_FLAG_FIPS_BYPASS) ? 1 : 0);
 #else
     md = EVP_get_digestbyname(to_openssl_alg(alg));
 #endif
@@ -807,14 +861,7 @@ unsigned char *cl_hash_data(const char *alg, const void *buf, size_t len, unsign
 #endif
 
 #if OPENSSL_VERSION_MAJOR >= 3
-    /* Bypass FIPS restrictions the OpenSSL 3.0 way */
-    ossl_ctx = OSSL_LIB_CTX_new();
-    if (NULL == ossl_ctx) {
-        cli_errmsg("cl_hash_data_ex: Failed to create new OpenSSL library context\n");
-        return NULL;
-    }
-
-    md = EVP_MD_fetch(ossl_ctx, to_openssl_alg(alg), "-fips");
+    md = cli_get_md(alg, 1);
 #else
     md = EVP_get_digestbyname(to_openssl_alg(alg));
 #endif
@@ -948,14 +995,7 @@ unsigned char *cl_hash_file_fd(int fd, const char *alg, unsigned int *olen)
     unsigned char *res;
 
 #if OPENSSL_VERSION_MAJOR >= 3
-    /* Bypass FIPS restrictions the OpenSSL 3.0 way */
-    ossl_ctx = OSSL_LIB_CTX_new();
-    if (NULL == ossl_ctx) {
-        cli_errmsg("cl_hash_data_ex: Failed to create new OpenSSL library context\n");
-        return NULL;
-    }
-
-    md = EVP_MD_fetch(ossl_ctx, to_openssl_alg(alg), "-fips");
+    md = cli_get_md(alg, 1);
 #else
     md = EVP_get_digestbyname(to_openssl_alg(alg));
 #endif
@@ -1852,14 +1892,7 @@ void *cl_hash_init(const char *alg)
 #endif
 
 #if OPENSSL_VERSION_MAJOR >= 3
-    /* Bypass FIPS restrictions the OpenSSL 3.0 way */
-    ossl_ctx = OSSL_LIB_CTX_new();
-    if (NULL == ossl_ctx) {
-        cli_errmsg("cl_hash_data_ex: Failed to create new OpenSSL library context\n");
-        return NULL;
-    }
-
-    md = EVP_MD_fetch(ossl_ctx, to_openssl_alg(alg), "-fips");
+    md = cli_get_md(alg, 1);
 #else
     md = EVP_get_digestbyname(to_openssl_alg(alg));
 #endif