erts: Only save the priority around setting IP_TOS

Every setsockopt went through a save and restore of both SO_PRIORITY
and IP_TOS, on the grounds that "if any other option is set after tos,
tos might be zeroed" - a comment carried over from the inet driver.
That costs two getsockopt calls and up to two more setsockopt calls on
every option set.

Only half of it holds. Setting IP_TOS does make the kernel derive a new
SO_PRIORITY from it, so that one option has to put the priority back.
Setting SO_PRIORITY leaves the tos alone, and so does setting anything
else: verified on Linux 7.0 with TCP_NODELAY, SO_RCVBUF and SO_LINGER,
none of which disturbed either value.

So do it only for IP_TOS, and only for the priority. Setting an option
on an accepted connection goes from four system calls to one. The
socket options a connection sets still look independent to the user,
which is what the dance was there for.

Author: lpgauth

erts/emulator/nifs/common/prim_socket_nif.c

@@ -11471,57 +11471,44 @@ int socket_setopt(int sock, int level, int opt,
     int res;
 
 #if  defined(IP_TOS) && defined(SOL_IP) && defined(SO_PRIORITY)
-    int          tmpIValPRIO = 0;
-    int          tmpIValTOS = 0;
-    int          resPRIO;
-    int          resTOS;
-    SOCKOPTLEN_T tmpArgSzPRIO = sizeof(tmpIValPRIO);
-    SOCKOPTLEN_T tmpArgSzTOS  = sizeof(tmpIValTOS);
-
-    resPRIO = sock_getopt(sock, SOL_SOCKET, SO_PRIORITY,
-                           &tmpIValPRIO, &tmpArgSzPRIO);
-    resTOS  = sock_getopt(sock, SOL_IP, IP_TOS,
-                          &tmpIValTOS, &tmpArgSzTOS);
+    /* Setting IP_TOS makes the kernel derive a new SO_PRIORITY from it,
+     * so that one option - and only that one - has to have the priority
+     * saved and put back to keep the two looking independent to the
+     * user. Setting anything else leaves both alone, and paying a
+     * getsockopt for each of them plus a setsockopt to restore on every
+     * option set was most of the system calls a connection made.
+     */
+    if ((level == SOL_IP) && (opt == IP_TOS)) {
+        int          savedPRIO;
+        int          resPRIO;
+        SOCKOPTLEN_T savedSzPRIO = sizeof(savedPRIO);
 
-    res = sock_setopt(sock, level, opt, optVal, optLen);
-    if (res == 0) {
+        resPRIO = sock_getopt(sock, SOL_SOCKET, SO_PRIORITY,
+                              &savedPRIO, &savedSzPRIO);
 
-        /* Ok, now we *maybe* need to "maybe" restore PRIO and TOS...
-         * maybe, possibly, ...
-         */
+        res = sock_setopt(sock, level, opt, optVal, optLen);
 
-        if (opt != SO_PRIORITY) {
-	    if ((opt != IP_TOS) && (resTOS == 0)) {
-		resTOS = sock_setopt(sock, SOL_IP, IP_TOS,
-                                     (void *) &tmpIValTOS,
-                                     tmpArgSzTOS);
-                res = resTOS;
-            }
-	    if ((res == 0) && (resPRIO == 0)) {
-		resPRIO = sock_setopt(sock, SOL_SOCKET, SO_PRIORITY,
-                                      &tmpIValPRIO,
-                                      tmpArgSzPRIO);
-
-                /* Some kernels set a SO_PRIORITY by default
-                 * that you are not permitted to reset,
-                 * silently ignore this error condition.
-                 */
-
-                if ((resPRIO != 0) && (sock_errno() == EPERM)) {
-                    res = 0;
-                } else {
-                    res = resPRIO;
-		}
-	    }
-	}
-    }
+        if ((res == 0) && (resPRIO == 0)) {
+            resPRIO = sock_setopt(sock, SOL_SOCKET, SO_PRIORITY,
+                                  &savedPRIO, savedSzPRIO);
 
-#else
+            /* Some kernels set a SO_PRIORITY by default
+             * that you are not permitted to reset,
+             * silently ignore this error condition.
+             */
 
-    res = sock_setopt(sock, level, opt, optVal, optLen);
+            if ((resPRIO != 0) && (sock_errno() == EPERM))
+                res = 0;
+            else
+                res = resPRIO;
+        }
 
+        return res;
+    }
 #endif
 
+    res = sock_setopt(sock, level, opt, optVal, optLen);
+
     return res;
 }