Harden X509 DER length handling in wolfSSL_X509_get_der and wolfSSL_i2d_X509

- src/x509.c: Guard wolfSSL_X509_get_der against derCert->length > INT_MAX, and reject derSz <= 0 in wolfSSL_i2d_X509.
- tests/api/test_ossl_x509_io.{c,h}: Add API coverage for the X509 DER length guards.

Author: ColtonWilley

src/x509.c

@@ -4523,6 +4523,10 @@ const byte* wolfSSL_X509_get_der(WOLFSSL_X509* x509, int* outSz)
     if (x509 == NULL || x509->derCert == NULL || outSz == NULL)
         return NULL;
 
+    if (x509->derCert->length > (word32)INT_MAX) {
+        return NULL;
+    }
+
     *outSz = (int)x509->derCert->length;
     return x509->derCert->buffer;
 }
@@ -8835,7 +8839,7 @@ int wolfSSL_i2d_X509(WOLFSSL_X509* x509, unsigned char** out)
     }
 
     der = wolfSSL_X509_get_der(x509, &derSz);
-    if (der == NULL) {
+    if (der == NULL || derSz <= 0) {
         WOLFSSL_LEAVE("wolfSSL_i2d_X509", MEMORY_E);
         return MEMORY_E;
     }

tests/api/test_ossl_x509_io.c

@@ -21,6 +21,8 @@
 
 #include <tests/unit.h>
 
+#include <limits.h>
+
 #ifdef NO_INLINE
     #include <wolfssl/wolfcrypt/misc.h>
 #else
@@ -72,6 +74,67 @@ int test_wolfSSL_i2d_X509(void)
     return EXPECT_RESULT();
 }
 
+int test_wolfSSL_X509_get_der_length_guards(void)
+{
+    EXPECT_DECLS;
+#if defined(OPENSSL_EXTRA) && defined(USE_CERT_BUFFERS_2048) && !defined(NO_RSA)
+    const unsigned char* cert_buf = server_cert_der_2048;
+    X509* cert = NULL;
+    int derSz = 0;
+    word32 origLen = 0;
+
+    ExpectNotNull(d2i_X509(&cert, &cert_buf, sizeof_server_cert_der_2048));
+    ExpectNotNull(cert);
+    ExpectNotNull(cert->derCert);
+
+    if (EXPECT_SUCCESS()) {
+        origLen = cert->derCert->length;
+        cert->derCert->length = ((word32)INT_MAX) + 1U;
+        ExpectNull(wolfSSL_X509_get_der(cert, &derSz));
+        cert->derCert->length = origLen;
+    }
+
+    X509_free(cert);
+#endif
+    return EXPECT_RESULT();
+}
+
+int test_wolfSSL_i2d_X509_der_length_guards(void)
+{
+    EXPECT_DECLS;
+#if defined(OPENSSL_EXTRA) && defined(USE_CERT_BUFFERS_2048) && !defined(NO_RSA)
+    const unsigned char* cert_buf = server_cert_der_2048;
+    unsigned char buf[4] = { 0x11, 0x22, 0x33, 0x44 };
+    const unsigned char origBuf[4] = { 0x11, 0x22, 0x33, 0x44 };
+    unsigned char* callerOut = buf;
+    X509* cert = NULL;
+    word32 origLen = 0;
+
+    ExpectNotNull(d2i_X509(&cert, &cert_buf, sizeof_server_cert_der_2048));
+    ExpectNotNull(cert);
+    ExpectNotNull(cert->derCert);
+
+    if (EXPECT_SUCCESS()) {
+        origLen = cert->derCert->length;
+
+        cert->derCert->length = ((word32)INT_MAX) + 1U;
+        ExpectIntEQ(i2d_X509(cert, &callerOut), MEMORY_E);
+        ExpectPtrEq(callerOut, buf);
+        ExpectIntEQ(XMEMCMP(buf, origBuf, sizeof(buf)), 0);
+
+        cert->derCert->length = 0;
+        ExpectIntEQ(i2d_X509(cert, &callerOut), MEMORY_E);
+        ExpectPtrEq(callerOut, buf);
+        ExpectIntEQ(XMEMCMP(buf, origBuf, sizeof(buf)), 0);
+
+        cert->derCert->length = origLen;
+    }
+
+    X509_free(cert);
+#endif
+    return EXPECT_RESULT();
+}
+
 int test_wolfSSL_PEM_read_X509(void)
 {
     EXPECT_DECLS;
@@ -244,4 +307,3 @@ int test_wolfSSL_PEM_write_bio_X509(void)
 #endif
     return EXPECT_RESULT();
 }
-

tests/api/test_ossl_x509_io.h

@@ -25,11 +25,17 @@
 #include <tests/api/api_decl.h>
 
 int test_wolfSSL_i2d_X509(void);
+int test_wolfSSL_X509_get_der_length_guards(void);
+int test_wolfSSL_i2d_X509_der_length_guards(void);
 int test_wolfSSL_PEM_read_X509(void);
 int test_wolfSSL_PEM_write_bio_X509(void);
 
 #define TEST_OSSL_X509_IO_DECLS                                         \
     TEST_DECL_GROUP("ossl_x509_io", test_wolfSSL_i2d_X509),             \
+    TEST_DECL_GROUP("ossl_x509_io",                                     \
+        test_wolfSSL_X509_get_der_length_guards),                       \
+    TEST_DECL_GROUP("ossl_x509_io",                                     \
+        test_wolfSSL_i2d_X509_der_length_guards),                       \
     TEST_DECL_GROUP("ossl_x509_io", test_wolfSSL_PEM_read_X509),        \
     TEST_DECL_GROUP("ossl_x509_io", test_wolfSSL_PEM_write_bio_X509)