Handle some IllegalArgument cases in readBytes

src/main/cpp/_nix_based/jssc.cpp

@@ -26,6 +26,7 @@
 #include <limits.h>
 #include <stdio.h>
 #include <fcntl.h>
+#include <new> // std::bad_alloc
 #include <string.h>
 #include <unistd.h>
 #include <string.h>
@@ -657,10 +658,31 @@ JNIEXPORT jbyteArray JNICALL Java_jssc_SerialNativeInterface_readBytes
   (JNIEnv *env, jobject, jlong portHandle, jint byteCount){
 
     int err;
-    jbyte *lpBuffer = new jbyte[byteCount];
+    jbyte *lpBuffer = NULL;
     jbyteArray returnArray = NULL;
     int byteRemains = byteCount;
 
+    if( byteCount < 0 ){
+        char emsg[32]; emsg[0] = '\0';
+        snprintf(emsg, sizeof emsg, "new byte[%d]", byteCount);
+        jclass exClz = env->FindClass("java/lang/IllegalArgumentException");
+        if( exClz != NULL ) env->ThrowNew(exClz, emsg);
+        returnArray = NULL; goto Finally;
+    }
+
+    try{
+        lpBuffer = new jbyte[byteCount];
+    }catch( const std::bad_alloc& ex ){
+        lpBuffer = NULL;
+    }
+    if( lpBuffer == NULL ){
+        char emsg[32]; emsg[0] = '\0';
+        snprintf(emsg, sizeof emsg, "new byte[%d]", byteCount);
+        jclass exClz = env->FindClass("java/lang/OutOfMemoryError");
+        if( exClz != NULL ) env->ThrowNew(exClz, emsg);
+        returnArray = NULL; goto Finally;
+    }
+
     while(byteRemains > 0) {
         int result = 0;
 
@@ -707,7 +729,7 @@ JNIEXPORT jbyteArray JNICALL Java_jssc_SerialNativeInterface_readBytes
     assert(env->ExceptionCheck() == JNI_FALSE);
 
 Finally:
-    delete[] lpBuffer;
+    if( lpBuffer != NULL ) delete[] lpBuffer;
     return returnArray;
 }
 

src/main/cpp/windows/jssc.cpp

@@ -276,17 +276,33 @@ JNIEXPORT jbyteArray JNICALL Java_jssc_SerialNativeInterface_readBytes
     HANDLE hComm = (HANDLE)portHandle;
     DWORD lpNumberOfBytesTransferred;
     DWORD lpNumberOfBytesRead;
+    jbyteArray returnArray = NULL;
     jbyte *lpBuffer = NULL;
+    OVERLAPPED *overlapped = NULL;
 
-    jbyteArray returnArray = env->NewByteArray(byteCount);
+    if( byteCount < 0 ){
+        /* Negative byteCount makes no sense -> Report to caller by exception. */
+        char exMsg[48]; exMsg[0] = '\0';
+        snprintf(exMsg, sizeof exMsg, "byteCount %d", byteCount);
+        jclass exClz = env->FindClass("java/lang/IllegalArgumentException");
+        if( exClz != NULL ) env->ThrowNew(exClz, exMsg);
+        goto Finally;
+    }
+
+    returnArray = env->NewByteArray(byteCount);
+    if( returnArray == NULL ) goto Finally;
 
     lpBuffer = (jbyte *)malloc(byteCount * sizeof(jbyte));
     if(lpBuffer == NULL){
-        // return an empty array
-        return returnArray;
+        /* Whops. Not enough memory. Let caller know through exception. */
+        char exMsg[32]; exMsg[0] = '\0';
+        snprintf(exMsg, sizeof exMsg, "malloc(%d)", byteCount);
+        jclass exClz = env->FindClass("java/lang/OutOfMemoryError");
+        if( exClz != NULL ) env->ThrowNew(exClz, exMsg);
+        goto Finally;
     }
 
-    OVERLAPPED *overlapped = new OVERLAPPED();
+    overlapped = new OVERLAPPED();
     overlapped->hEvent = CreateEventA(NULL, true, false, NULL);
     if(ReadFile(hComm, lpBuffer, (DWORD)byteCount, &lpNumberOfBytesRead, overlapped)){
         env->SetByteArrayRegion(returnArray, 0, byteCount, lpBuffer);
@@ -302,9 +318,12 @@ JNIEXPORT jbyteArray JNICALL Java_jssc_SerialNativeInterface_readBytes
         jclass exClz = env->FindClass("java/lang/IllegalArgumentException");
         if( exClz != NULL ) env->ThrowNew(exClz, "EBADF");
     }
-    CloseHandle(overlapped->hEvent);
-    delete overlapped;
-    free(lpBuffer);
+Finally:
+    if( overlapped != NULL ){
+        CloseHandle(overlapped->hEvent);
+        delete overlapped;
+    }
+    if( lpBuffer != NULL ) free(lpBuffer);
     return returnArray;
 }
 

src/test/java/jssc/SerialNativeInterfaceTest.java

@@ -94,4 +94,34 @@ public void throwsNpeIfPassedBufferIsNull() throws Exception {
         new SerialNativeInterface().writeBytes(1, null);
     }
 
+    @Test
+    public void throwsIaeIfCountNegative() throws Exception {
+        SerialNativeInterface testTarget = new SerialNativeInterface();
+        byte[] ret;
+        try{
+            ret = testTarget.readBytes(0, -42);
+            fail("Where's the exception?");
+        }catch( IllegalArgumentException ex ){
+            assertTrue(ex.getMessage().contains("-42"));
+        }
+    }
+
+    @Test
+    @org.junit.Ignore("This test only makes sense if it is run in a situation"
+        +" where large memory allocations WILL fail (for example you could use"
+        +" a virtual machine with low memory available). Because on regular"
+        +" machines allocating 2GiB of RAM is not a problem at all and so the"
+        +" test run will just happily wait infinitely for those 2GiB to arrive"
+        +" at the stdin fd. Feel free to remove this test if you think it"
+        +" doesn't make sense to have it here.")
+    public void throwsOOMExIfRequestTooLarge() throws Exception {
+        SerialNativeInterface testTarget = new SerialNativeInterface();
+        try{
+            byte[] ret = testTarget.readBytes(0, Integer.MAX_VALUE);
+            fail("Where's the exception?");
+        }catch( OutOfMemoryError ex ){
+            assertTrue(ex.getMessage().contains(String.valueOf(Integer.MAX_VALUE)));
+        }
+    }
+
 }