Don't call SSL_shutdown() when receiving SSL_ERROR_SYSCALL or SSL_ERROR_SSL (#3577)

Author: trengginas

pjlib/src/pj/ssl_sock_imp_common.c

@@ -244,6 +244,7 @@ static void ssl_close_sockets(pj_ssl_sock_t *ssock)
 static pj_bool_t on_handshake_complete(pj_ssl_sock_t *ssock, 
                                        pj_status_t status)
 {
+    ssock->handshake_status = status;
     /* Cancel handshake timer */
     if (ssock->timer.id == TIMER_HANDSHAKE_TIMEOUT) {
         pj_timer_heap_cancel(ssock->param.timer_heap, &ssock->timer);

pjlib/src/pj/ssl_sock_imp_common.h

@@ -112,6 +112,7 @@ struct pj_ssl_sock_t
     pj_ioqueue_op_key_t   shutdown_op_key;
     pj_timer_entry        timer;
     pj_status_t           verify_status;
+    pj_status_t           handshake_status;
 
     pj_bool_t             is_closing;
     unsigned long         last_err;

pjlib/src/pj/ssl_sock_ossl.c

@@ -1714,13 +1714,20 @@ static void ssl_reset_sock_state(pj_ssl_sock_t *ssock)
      * Avoid calling SSL_shutdown() if handshake wasn't completed.
      * OpenSSL 1.0.2f complains if SSL_shutdown() is called during an
      * SSL handshake, while previous versions always return 0.
+     * Call SSL_shutdown() when there is a timeout handshake failure or
+     * the last error is not SSL_ERROR_SYSCALL and not SSL_ERROR_SSL.
      */
     if (ossock->ossl_ssl && SSL_in_init(ossock->ossl_ssl) == 0) {
-        int ret = SSL_shutdown(ossock->ossl_ssl);
-        if (ret == 0) {
-            /* SSL_shutdown will potentially trigger a bunch of
-             * data to dump to the socket */
-            post_unlock_flush_circ_buf = 1;
+        if (ssock->handshake_status == PJ_ETIMEDOUT ||
+            (ssock->last_err != SSL_ERROR_SYSCALL &&
+             ssock->last_err != SSL_ERROR_SSL))
+        {
+            int ret = SSL_shutdown(ossock->ossl_ssl);
+            if (ret == 0) {
+                /* SSL_shutdown will potentially trigger a bunch of
+                 * data to dump to the socket */
+                post_unlock_flush_circ_buf = 1;
+            }
         }
     }