libckteec: fix integer overflow in PKCS#11 serializer

Add overflow check in serialize() to prevent size_t wraparound when
computing new buffer length. A crafted ulValueLen could cause *blen + len
to wrap, leading to a small realloc followed by an out-of-bounds memcpy.

Also add a bounds check in serialize_ck_attribute() for the
CKA_ALLOWED_MECHANISMS path where n * sizeof(uint32_t) could overflow
the uint32_t pkcs11_size, resulting in an undersized malloc.

Fixes: 85a7ea7 ("libckteec: introduce helpers for serializing data")
Signed-off-by: Minghao Cheng <m@minhal.me>

Author: bestminhal

libckteec/src/serialize_ck.c

@@ -173,6 +173,8 @@ static CK_RV serialize_ck_attribute(struct serializer *obj, CK_ATTRIBUTE *attr)
 		return serialize_indirect_attribute(obj, attr);
 	case CKA_ALLOWED_MECHANISMS:
 		n = attr->ulValueLen / sizeof(CK_ULONG);
+		if (n > UINT32_MAX / sizeof(uint32_t))
+			return CKR_ARGUMENTS_BAD;
 		pkcs11_size = n * sizeof(uint32_t);
 		mech_buf = malloc(pkcs11_size);
 		if (!mech_buf)

libckteec/src/serializer.c

@@ -45,7 +45,12 @@ void release_serial_object(struct serializer *obj)
 static CK_RV serialize(char **bstart, size_t *blen, void *data, size_t len)
 {
 	size_t nlen = *blen + len;
-	char *buf = realloc(*bstart, nlen);
+	char *buf = NULL;
+
+	if (nlen < *blen)
+		return CKR_HOST_MEMORY;
+
+	buf = realloc(*bstart, nlen);
 
 	if (!buf)
 		return CKR_HOST_MEMORY;