Merge pull request #10297 from willus10245/fix-odbc-memory-allocation-error

cap max column sizes for mssql and oracle

OTP-20328

Author: IngelaAndin

lib/odbc/c_src/odbcserver.c

@@ -28,7 +28,7 @@
   database using the Microsoft ODBC API. The c-process is implemented
   using two threads the supervisor thread and the database handler thread.
   If the database thread should hang erlang can close the c-process down
-  by sendig a shutdown request to the supervisor thread.
+  by sending a shutdown request to the supervisor thread.
 
   Erlang will start this c-process as a port-program and send information
   regarding inet-port numbers through the erlang-port.
@@ -62,7 +62,10 @@
    they are converted to string values.
 
    [?OPEN_CONNECTION, C_AutoCommitMode, C_TraceDriver, C_SrollableCursors,
-   C_TupelRow, BinaryStrings, ConnectionStr]
+   C_TupelRow, BinaryStrings, ExtendedErrors, 255, 1,
+   MaxLongColumnSizeOctets (4 bytes, big-endian, 0 = default), ConnectionStr]
+   Legacy layout: same first six bytes then ConnectionStr; optional tuning via
+   ERL_ODBC_MAX_LONG_COLUMN_SIZE.
    [?CLOSE_CONNECTION]		     
    [?COMMIT_TRANSACTION, CommitMode]
    [?QUERY, SQLQuery]
@@ -80,6 +83,7 @@
    C_SrollableCursors - ?ON | ?OFF
    C_TupelRow -  - ?ON | ?OFF
    BinaryStrings - ?ON | ?OFF
+   ExtendedErrors - ?ON | ?OFF
    ConnectionStr -  String
    CommitMode -  ?COMMIT | ?ROLLBACK
    SQLQuery  - String
@@ -105,6 +109,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include <stdio.h>
+#include <errno.h>
 
 #ifdef UNIX
 #include <unistd.h>
@@ -141,11 +146,12 @@ DWORD WINAPI database_handler(const char *port);
 #else
 void database_handler(const char *port);
 #endif
-static db_result_msg handle_db_request(byte *reqstring, db_state *state);
+static db_result_msg handle_db_request(byte *reqstring, size_t msg_len,
+                                       db_state *state);
 static void supervise(const char *port);
 /* ----------------- ODBC functions --------------------------------------*/
 
-static db_result_msg db_connect(byte *connStrIn, db_state *state);
+static db_result_msg db_connect(byte *args, size_t args_len, db_state *state);
 static db_result_msg db_close_connection(db_state *state);
 static db_result_msg db_end_tran(byte compleationtype, db_state *state);
 static db_result_msg db_query(byte *sql, db_state *state);
@@ -193,7 +199,7 @@ static byte * receive_erlang_port_msg(void);
 #ifdef WIN32
 static SOCKET connect_to_erlang(const char *port); 
 static void send_msg(db_result_msg *msg, SOCKET socket);
-static byte *receive_msg(SOCKET socket);
+static byte *receive_msg(SOCKET socket, size_t *msg_len_out);
 static Boolean receive_msg_part(SOCKET socket,
 				byte * buffer, size_t msg_len);
 static Boolean send_msg_part(SOCKET socket, byte * buffer, size_t msg_len);
@@ -202,7 +208,7 @@ static void init_winsock(void);
 #elif defined(UNIX)
 static int connect_to_erlang(const char *port);
 static void send_msg(db_result_msg *msg, int socket);
-static byte *receive_msg(int socket);
+static byte *receive_msg(int socket, size_t *msg_len_out);
 static Boolean receive_msg_part(int socket, byte * buffer, size_t msg_len);
 static Boolean send_msg_part(int socket, byte * buffer, size_t msg_len);
 static void close_socket(int socket); 
@@ -333,7 +339,7 @@ void supervise(const char *port) {
 #endif
 
     socket = connect_to_erlang(port);
-    msg = receive_msg(socket);
+    msg = receive_msg(socket, NULL);
 
     if(msg[0] == SHUTDOWN) {
 	reason = EXIT_SUCCESS;
@@ -357,7 +363,7 @@ DWORD WINAPI database_handler(const char *port)
     byte *request_buffer = NULL;
     db_state state =
     {NULL, NULL, NULL, NULL, 0, {NULL, 0, 0},
-     FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE};
+     FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, (SQLULEN)0};
     byte request_id;
 #ifdef WIN32
     SOCKET socket;
@@ -369,23 +375,26 @@ DWORD WINAPI database_handler(const char *port)
     socket = connect_to_erlang(port);
 
     do {      
-	request_buffer = receive_msg(socket);
+        {
+            size_t req_len;
+            request_buffer = receive_msg(socket, &req_len);
 
-	request_id = request_buffer[0];
-	msg = handle_db_request(request_buffer, &state);
+            request_id = request_buffer[0];
+            msg = handle_db_request(request_buffer, req_len, &state);
+        }
+
+        send_msg(&msg, socket); /* Send answer to erlang */
+
+        if (msg.dyn_alloc) {
+            ei_x_free(&(state.dynamic_buffer));
+        } else {
+            free(msg.buffer);
+            msg.buffer = NULL;
+        }
+
+        free(request_buffer);
+        request_buffer = NULL;
 
-	send_msg(&msg, socket); /* Send answer to erlang */
-	    
-	if (msg.dyn_alloc) {
-	    ei_x_free(&(state.dynamic_buffer));
-	} else {
-	    free(msg.buffer);
-	    msg.buffer = NULL;
-	}   
-	
-	free(request_buffer);
-	request_buffer = NULL;
-    
     } while(request_id != CLOSE_CONNECTION);
 
     shutdown(socket, 2);
@@ -400,47 +409,78 @@ DWORD WINAPI database_handler(const char *port)
 /* Description: Calls the appropriate function to handle the database
    request received from the erlang-process. Returns a message to send back
    to erlang. */
-static db_result_msg handle_db_request(byte *reqstring, db_state *state)
+static db_result_msg handle_db_request(byte *reqstring, size_t msg_len,
+                                       db_state *state)
 {
     byte *args;
     byte request_id;
+    size_t args_len;
 
     /* First byte is an index that identifies the requested command the
        rest is the argument string. */
     request_id = reqstring[0]; 
     args = reqstring + sizeof(byte);
+    args_len = (msg_len > sizeof(byte)) ? (msg_len - sizeof(byte)) : 0;
 
     switch(request_id) {
     case OPEN_CONNECTION:
-	return db_connect(args, state); 
+        return db_connect(args, args_len, state);
     case CLOSE_CONNECTION:
-	return db_close_connection(state);
+        return db_close_connection(state);
     case COMMIT_TRANSACTION:
-	if(args[0] == COMMIT) {
-	    return db_end_tran((byte)SQL_COMMIT, state);
-	} else { /* args[0] == ROLLBACK */
-	    return db_end_tran((byte)SQL_ROLLBACK, state);
-	}
+        if(args[0] == COMMIT) {
+            return db_end_tran((byte)SQL_COMMIT, state);
+        } else { /* args[0] == ROLLBACK */
+            return db_end_tran((byte)SQL_ROLLBACK, state);
+        }
     case QUERY:
-	return db_query(args, state);
+        return db_query(args, state);
     case SELECT_COUNT:
-	return db_select_count(args, state);
+        return db_select_count(args, state);
     case SELECT:
-	return db_select(args, state);
+        return db_select(args, state);
     case PARAM_QUERY:
-	return db_param_query(args, state);
+        return db_param_query(args, state);
     case DESCRIBE:
-	return db_describe_table(args, state);
+        return db_describe_table(args, state);
     default:
-	DO_EXIT(EXIT_FAILURE); /* Should not happen */
-    }  
+        DO_EXIT(EXIT_FAILURE); /* Should not happen */
+    }
 }
 
 /* ----------------- ODBC-functions  ----------------------------------*/
+
+static SQLULEN
+clamp_max_long_col_size(SQLULEN value)
+{
+    if (value == (SQLULEN)0)
+        return (SQLULEN)DEFAULT_MAX_LONG_COL_SIZE;
+    if (value > (SQLULEN)HARD_MAX_LONG_COL_SIZE)
+        return (SQLULEN)HARD_MAX_LONG_COL_SIZE;
+    return value;
+}
+
+static SQLULEN
+max_long_col_size_from_env(void)
+{
+    const char *e = getenv("ERL_ODBC_MAX_LONG_COLUMN_SIZE");
+    unsigned long v;
+    char *end = NULL;
+
+    if (e == NULL || e[0] == '\0')
+        return (SQLULEN)0;
+    errno = 0;
+    v = strtoul(e, &end, 10);
+    if (end == e || *end != '\0')
+        return (SQLULEN)0;
+    if (errno == ERANGE)
+        return (SQLULEN)0;
+    return (SQLULEN)v;
+}
 
 /* Description: Tries to open a connection to the database using
    <connStrIn>, returns a message indicating the outcome. */
-static db_result_msg db_connect(byte *args, db_state *state)
+static db_result_msg db_connect(byte *args, size_t args_len, db_state *state)
 {
     /*
      * Danil Onishchenko aka RubberCthulhu, alevandal@gmail.com. 2013.01.09.
@@ -465,14 +505,32 @@ static db_result_msg db_connect(byte *args, db_state *state)
     int erl_auto_commit_mode, erl_trace_driver,
 	    use_srollable_cursors, tuple_row_state, binary_strings,
 	    extended_errors;
+    SQLULEN long_col_cfg;
 
+    if (args_len < 6)
+        DO_EXIT(EXIT_FAILURE);
+
     erl_auto_commit_mode = args[0];
     erl_trace_driver = args[1];
     use_srollable_cursors = args[2];
     tuple_row_state = args[3];
     binary_strings = args[4];
     extended_errors = args[5];
-    connStrIn = args + 6 * sizeof(byte);
+
+    if (args_len >= 13 && args[6] == OPEN_CONNECTION_PROTO_V1_A &&
+        args[7] == OPEN_CONNECTION_PROTO_V1_B) {
+        long_col_cfg = ((SQLULEN)args[8] << 24) | ((SQLULEN)args[9] << 16) |
+            ((SQLULEN)args[10] << 8) | (SQLULEN)args[11];
+        connStrIn = args + 12;
+        max_long_column_size(state) = clamp_max_long_col_size(long_col_cfg);
+    } else {
+        connStrIn = args + 6;
+        max_long_column_size(state) =
+            clamp_max_long_col_size(max_long_col_size_from_env());
+    }
+
+    if ((size_t)(connStrIn - args) >= args_len)
+        DO_EXIT(EXIT_FAILURE);
 
     if(tuple_row_state == ON) {
 	    tuple_row(state) = TRUE;  
@@ -1325,9 +1383,9 @@ static db_result_msg encode_column_name_list(SQLSMALLINT num_of_columns,
     SQLULEN size;
 
     msg = encode_empty_message();
-    
+
     ei_x_encode_list_header(&dynamic_buffer(state), num_of_columns);
-  
+
     for (i = 0; i < num_of_columns; ++i) {
 
 	if(!sql_success(SQLDescribeCol(statement_handle(state),
@@ -1337,32 +1395,37 @@ static db_result_msg encode_column_name_list(SQLSMALLINT num_of_columns,
 				       &nullable)))
 	    DO_EXIT(EXIT_DESC);
 
-	if(size == 0 && (sql_type == SQL_LONGVARCHAR || sql_type == SQL_LONGVARBINARY || sql_type == SQL_WLONGVARCHAR))
-	    size = MAXCOLSIZE;
-    
+        if(sql_type == SQL_LONGVARCHAR || sql_type == SQL_LONGVARBINARY || sql_type == SQL_WLONGVARCHAR) {
+            if(size == 0) {
+                size = max_long_column_size(state);
+            } else if(size > max_long_column_size(state)) {
+                size = max_long_column_size(state);
+            }
+        }
+
 	(columns(state)[i]).type.decimal_digits = dec_digits;
 	(columns(state)[i]).type.sql = sql_type;
 	(columns(state)[i]).type.col_size = size;
-      
+
 	msg = map_sql_2_c_column(&columns(state)[i], state);
 	if (msg.length > 0) {
 	    return msg; /* An error has occurred */
 	} else {
 	    if (columns(state)[i].type.len > 0) {
 		columns(state)[i].buffer =
 		    (char *)safe_malloc(columns(state)[i].type.len);
-	
+
 		if (columns(state)[i].type.c == SQL_C_BINARY) {
 		    /* retrieved later by retrive_binary_data */
 		} else {
 		    if(!sql_success(
-			SQLBindCol
-			(statement_handle(state),
-			 (SQLSMALLINT)(i+1),
-			 columns(state)[i].type.c,
-			 columns(state)[i].buffer,
-			 columns(state)[i].type.len,
-			 &columns(state)[i].type.strlen_or_indptr)))
+                                    SQLBindCol
+                                    (statement_handle(state),
+                                     (SQLSMALLINT)(i+1),
+                                     columns(state)[i].type.c,
+                                     columns(state)[i].buffer,
+                                     columns(state)[i].type.len,
+                                     &columns(state)[i].type.strlen_or_indptr)))
 			DO_EXIT(EXIT_BIND);
 		}
 		ei_x_encode_string_len(&dynamic_buffer(state),
@@ -1395,9 +1458,9 @@ static db_result_msg encode_value_list(SQLSMALLINT num_of_columns,
 	result = SQLFetch(statement_handle(state)); 
 
 	if (result == SQL_NO_DATA) /* Reached end of result set */
-	{
-	    break;
-	}
+            {
+                break;
+            }
 
 	ei_x_encode_list_header(&dynamic_buffer(state), 1);
 
@@ -1956,9 +2019,9 @@ static void close_socket(int socket)
 #endif
 
 #ifdef WIN32
-static byte * receive_msg(SOCKET socket) 
+static byte * receive_msg(SOCKET socket, size_t *msg_len_out)
 #elif defined(UNIX)
-static byte * receive_msg(int socket) 
+static byte * receive_msg(int socket, size_t *msg_len_out)
 #endif
 {
     byte lengthstr[LENGTH_INDICATOR_SIZE];
@@ -1979,6 +2042,9 @@ static byte * receive_msg(int socket)
 	DO_EXIT(EXIT_SOCKET_RECV_BODY);
     }
 
+    if (msg_len_out != NULL)
+        *msg_len_out = msg_len;
+
     return buffer;
 }
 

lib/odbc/c_src/odbcserver.h

@@ -24,7 +24,21 @@
 
 /* ----------------------------- CONSTANTS ------------------------------*/
 
-#define MAXCOLSIZE 8001
+/* Default ODBC bind buffer size (bytes) for SQL_LONGVARCHAR,
+ * SQL_LONGVARBINARY, and SQL_WLONGVARCHAR when the driver reports size 0 or an
+ * oversized display size (e.g. SQL Server varchar(max)). Override per
+ * connection from Erlang (see max_long_column_size) or with environment
+ * variable ERL_ODBC_MAX_LONG_COLUMN_SIZE for legacy clients. Larger values
+ * use more memory per column per bound row. */
+#define DEFAULT_MAX_LONG_COL_SIZE 8001
+/* Hard upper bound to avoid runaway allocation (256 MiB per column). */
+#define HARD_MAX_LONG_COL_SIZE (256U * 1024U * 1024U)
+/* Extended OPEN_CONNECTION layout: these two bytes follow the six option
+ * bytes, then four big-endian size octets, then the connection string.
+ * (Single-byte 0xFF alone could match a rare first character of a legacy
+ * connection string.) */
+#define OPEN_CONNECTION_PROTO_V1_A 0xFFU
+#define OPEN_CONNECTION_PROTO_V1_B 0x01U
 #define MAX_ERR_MSG 1024
 #define ERRMSG_HEADR_SIZE 20
 #define MAX_CONN_STR_OUT 1024
@@ -184,6 +198,7 @@ typedef struct {
     Boolean param_query;
     Boolean out_params;
     Boolean extended_errors;
+    SQLULEN max_long_column_size;
 } db_state;
 
 typedef enum {
@@ -205,3 +220,4 @@ typedef enum {
 #define out_params(db_state) (db_state -> out_params)
 #define extended_errors(db_state) (db_state -> extended_errors)
 #define extended_error(db_state, errorcode) ( extended_errors(state) ? ((char *)errorcode) : NULL )
+#define max_long_column_size(db_state) ((db_state)->max_long_column_size)