Rebase using version 3.20+

Author: davidBar-On

configure.ac

@@ -253,6 +253,10 @@ AC_CHECK_FUNCS([sendfile])
 # connections.
 AC_CHECK_FUNCS([getline])
 
+# Check for sendmmsg/recvmmsg support for better throughput.
+AC_CHECK_FUNCS([sendmmsg recvmmsg],
+	       AC_DEFINE([HAVE_SEND_RECVMMSG], [1], [Have sendmmsg/recvmmsg functions.]))
+
 # Check for packet pacing socket option (Linux only for now).
 AC_CACHE_CHECK([SO_MAX_PACING_RATE socket option],
 [iperf3_cv_header_so_max_pacing_rate],

src/iperf.h

@@ -91,6 +91,16 @@ struct iperf_sctp_info
     uint32_t cwnd;
 };
 
+#ifndef HAVE_SENDMMSG
+struct dummy_mmsghdr  {
+    struct msghdr msg_hdr;  /* Message header */
+    unsigned int  msg_len;  /* Number of received bytes for header */
+};
+#endif /* HAVE_SENDMMSG */
+
+/* macro to calculate buffer sizes used for a stream (to support hse of send/recvmmsg) */
+#define STREAM_BUFSIZE(prefix) (((prefix)->settings->blksize) * (((prefix)->settings->send_recvmmsg == 0 || (prefix)->settings->burst == 0)? 1 : (prefix)->settings->burst))
+
 struct iperf_interval_results
 {
     atomic_iperf_size_t bytes_transferred; /* bytes transferred in this interval */
@@ -186,6 +196,7 @@ struct iperf_settings
     int	      connect_timeout;	    /* socket connection timeout, in ms */
     int       idle_timeout;         /* server idle time timeout */
     unsigned int snd_timeout; /* Timeout for sending tcp messages in active mode, in us */
+    int       send_recvmmsg;        /* whether sendmmsg/recvmmsg should be used - mainly per the -Z option */
     struct iperf_time rcv_timeout;  /* Timeout for receiving messages in active mode, in us */
     int       cntl_ka;              /* Use Control TCP connection Keepalive */
     int       cntl_ka_keepidle;     /* Control TCP connection Keepalive idle time (TCP_KEEPIDLE) */
@@ -237,6 +248,16 @@ struct iperf_stream
     int64_t   omitted_outoforder_packets;
     int64_t   cnt_error;
     int64_t   omitted_cnt_error;
+    
+#ifdef HAVE_SENDMMSG
+    struct mmsghdr *msg; /* For some reason, just struct and not a pointer failes on "field ‘msg’ has incomplete type" */
+#else
+    struct dummy_mmsghdr *msg;
+#endif /* HAVE_SENDMMSG */
+    struct    iovec *pmsg_iov; /* Pointer to the array of iov_msg per mmsg message */
+    int       sendmmsg_buffered_packets_count; /* number of buffered packets for sendmmsg */
+    char      *pbuf; /* Pointer to current vailable space in buffer */
+
     uint64_t  target;
 
     struct sockaddr_storage local_addr;

src/iperf_api.c

@@ -43,6 +43,7 @@
 #include <fcntl.h>
 #include <sys/socket.h>
 #include <sys/types.h>
+#include <sys/uio.h>
 #include <netinet/in.h>
 #include <arpa/inet.h>
 #include <netdb.h>
@@ -101,6 +102,7 @@ static int JSON_write(int fd, cJSON *json);
 static void print_interval_results(struct iperf_test *test, struct iperf_stream *sp, cJSON *json_interval_streams);
 static cJSON *JSON_read(int fd, int max_size);
 static int JSONStream_Output(struct iperf_test *test, const char* event_name, cJSON* obj);
+void iperf_init_send_recvmmsg(struct iperf_test *test);
 
 
 /*************************** Print usage functions ****************************/
@@ -1971,6 +1973,9 @@ iperf_parse_arguments(struct iperf_test *test, int argc, char **argv)
 	    1 : round(test->settings->bitrate_limit_interval/test->stats_interval) );
     }
 
+    /* Set whether send/recvmmsg is supported along with related settings */
+    iperf_init_send_recvmmsg(test);
+
     /* Show warning if JSON output is used with explicit report format */
     if ((test->json_output) && (test->settings->unit_format != 'a')) {
         warning("Report format (-f) flag ignored with JSON output (-J)");
@@ -2335,6 +2340,9 @@ iperf_exchange_parameters(struct iperf_test *test)
             }
             return -1;
         }
+        
+        /* Set whether send/recvmmsg is supported along with related settings */
+        iperf_init_send_recvmmsg(test);
 
 #if defined(HAVE_SSL)
         if (test_is_authorized(test) < 0){
@@ -3255,6 +3263,8 @@ iperf_defaults(struct iperf_test *testp)
     testp->json_callback = NULL;
 
 
+    testp->zerocopy = 0;
+    testp->settings->send_recvmmsg = 0;
     memset(testp->cookie, 0, COOKIE_SIZE);
 
     testp->multisend = 10;	/* arbitrary */
@@ -3555,6 +3565,7 @@ iperf_reset_test(struct iperf_test *test)
     test->settings->dont_fragment = 0;
     test->zerocopy = 0;
     test->settings->skip_rx_copy = 0;
+    test->settings->send_recvmmsg = 0;
 
 #if defined(HAVE_SSL)
     if (test->settings->authtoken) {
@@ -3616,6 +3627,8 @@ iperf_reset_stats(struct iperf_test *test)
         sp->omitted_cnt_error = sp->cnt_error;
         sp->omitted_outoforder_packets = sp->outoforder_packets;
 	sp->jitter = 0;
+	sp->sendmmsg_buffered_packets_count = 0;
+        sp->pbuf = sp->buffer;
 	rp = sp->result;
         rp->bytes_sent_omit = rp->bytes_sent;
         rp->bytes_received = 0;
@@ -4694,9 +4707,15 @@ void
 iperf_free_stream(struct iperf_stream *sp)
 {
     struct iperf_interval_results *irp, *nirp;
+    int r;
 
     /* XXX: need to free interval list too! */
-    munmap(sp->buffer, sp->test->settings->blksize);
+    r = munmap(sp->buffer, STREAM_BUFSIZE(sp->test));
+    if (r < 0)
+        printf("Failed to release stream buffer at %p with size %d  - errno=%d: %s)\n", sp->buffer, STREAM_BUFSIZE(sp->test), errno, strerror(errno));
+    else if (sp->test->debug)
+        printf("Successfully released stream buffer at %p with size %d\n", sp->buffer, STREAM_BUFSIZE(sp->test));
+
     close(sp->buffer_fd);
     if (sp->diskfile_fd >= 0)
 	close(sp->diskfile_fd);
@@ -4707,6 +4726,11 @@ iperf_free_stream(struct iperf_stream *sp)
     free(sp->result);
     if (sp->send_timer != NULL)
 	tmr_cancel(sp->send_timer);
+
+    if (sp->pmsg_iov != NULL)
+        free(sp->pmsg_iov);
+    if (sp->msg != NULL)
+        free(sp->msg);
     free(sp);
 }
 
@@ -4716,6 +4740,9 @@ iperf_new_stream(struct iperf_test *test, int s, int sender)
 {
     struct iperf_stream *sp;
     int ret = 0;
+    int burst;
+    int i;
+    char *pbuf;
 
     char template[1024];
     if (test->tmp_template) {
@@ -4774,19 +4801,50 @@ iperf_new_stream(struct iperf_test *test, int s, int sender)
         free(sp);
         return NULL;
     }
-    if (ftruncate(sp->buffer_fd, test->settings->blksize) < 0) {
+    if (ftruncate(sp->buffer_fd, STREAM_BUFSIZE(sp)) < 0) {
         i_errno = IECREATESTREAM;
         free(sp->result);
         free(sp);
         return NULL;
     }
-    sp->buffer = (char *) mmap(NULL, test->settings->blksize, PROT_READ|PROT_WRITE, MAP_SHARED, sp->buffer_fd, 0);
+
+    sp->buffer = (char *) mmap(NULL, STREAM_BUFSIZE(sp), PROT_READ|PROT_WRITE, MAP_PRIVATE, sp->buffer_fd, 0);
     if (sp->buffer == MAP_FAILED) {
         i_errno = IECREATESTREAM;
         free(sp->result);
         free(sp);
         return NULL;
     }
+    else if (sp->test->debug)
+        printf("Successfully allocated stream buffer at addr %p with length %d\n", sp->buffer, STREAM_BUFSIZE(sp));
+
+    /* Allocate and initialize pointers for the messages in the buffer for UDP multi-msg send/receive */
+    if (test->protocol->id == Pudp) {
+        burst = (sp->settings->send_recvmmsg == 0)? 1 : sp->settings->burst;
+        sp->msg = malloc(burst * sizeof(*sp->msg));
+        if (sp->msg == NULL) {
+            i_errno = IECREATESTREAM;
+            iperf_free_stream(sp);
+            return NULL;
+        }
+        memset(sp->msg, 0, burst * sizeof(*sp->msg));
+
+        sp->pmsg_iov = malloc(burst * sizeof(struct iovec));
+        if (sp->pmsg_iov == NULL) {
+            i_errno = IECREATESTREAM;
+            iperf_free_stream(sp);
+            return NULL;
+        }
+        memset(sp->pmsg_iov, 0, burst * sizeof(struct iovec));
+
+        for (i = 0, pbuf = sp->buffer; i < burst; i++, pbuf += sp->settings->blksize) {
+            sp->msg[i].msg_hdr.msg_iov = &sp->pmsg_iov[i];
+            sp->msg[i].msg_hdr.msg_iov->iov_base = pbuf;
+            sp->msg[i].msg_hdr.msg_iov->iov_len = sp->settings->blksize;
+            sp->msg[i].msg_hdr.msg_iovlen = 1;
+        }
+    }
+
     sp->pending_size = 0;
 
     /* Set socket */
@@ -4795,11 +4853,14 @@ iperf_new_stream(struct iperf_test *test, int s, int sender)
     sp->snd = test->protocol->send;
     sp->rcv = test->protocol->recv;
 
+    sp->sendmmsg_buffered_packets_count = 0;
+    sp->pbuf = sp->buffer;
+
     if (test->diskfile_name != (char*) 0) {
 	sp->diskfile_fd = open(test->diskfile_name, sender ? O_RDONLY : (O_WRONLY|O_CREAT|O_TRUNC), S_IRUSR|S_IWUSR);
 	if (sp->diskfile_fd == -1) {
 	    i_errno = IEFILE;
-            munmap(sp->buffer, sp->test->settings->blksize);
+            munmap(sp->buffer, STREAM_BUFSIZE(sp));
             free(sp->result);
             free(sp);
 	    return NULL;
@@ -4813,13 +4874,13 @@ iperf_new_stream(struct iperf_test *test, int s, int sender)
 
     /* Initialize stream */
     if (test->repeating_payload)
-        fill_with_repeating_pattern(sp->buffer, test->settings->blksize);
+        fill_with_repeating_pattern(sp->buffer, STREAM_BUFSIZE(sp));
     else
-        ret = readentropy(sp->buffer, test->settings->blksize);
+        ret = readentropy(sp->buffer, STREAM_BUFSIZE(sp));
 
     if ((ret < 0) || (iperf_init_stream(sp, test) < 0)) {
         close(sp->buffer_fd);
-        munmap(sp->buffer, sp->test->settings->blksize);
+        munmap(sp->buffer, STREAM_BUFSIZE(sp));
         free(sp->result);
         free(sp);
         return NULL;
@@ -4972,7 +5033,7 @@ diskfile_send(struct iperf_stream *sp)
 
     /* if needed, read enough data from the disk to fill up the buffer */
     if (sp->diskfile_left < sp->test->settings->blksize && !sp->test->done) {
-    	r = read(sp->diskfile_fd, sp->buffer, sp->test->settings->blksize -
+    	r = read(sp->diskfile_fd, sp->buffer, STREAM_BUFSIZE(sp) -
     		 sp->diskfile_left);
         buffer_left += r;
     	rtot += r;
@@ -4982,8 +5043,8 @@ diskfile_send(struct iperf_stream *sp)
 
         // If the buffer doesn't contain a full buffer at this point,
         // adjust the size of the data to send.
-        if (buffer_left != sp->test->settings->blksize) {
-            if (sp->test->debug)
+        if (buffer_left != STREAM_BUFSIZE(sp)) {
+            if (sp->test->debug) 
                 printf("possible eof\n");
             // setting data size to be sent,
             // which is less than full block/buffer size
@@ -5572,3 +5633,27 @@ iperf_set_control_keepalive(struct iperf_test *test)
     return 0;
 }
 #endif //HAVE_TCP_KEEPALIVE
+
+/*
+ * Initialize whether sed/recvmmsg is supported for the test,
+ * along with initializing other related settings.
+ */
+void
+iperf_init_send_recvmmsg(struct iperf_test *test)
+{
+#ifdef HAVE_SENDMMSG
+    /* Set UDP `send_recvmmsg` from `zerocopy` settings (not used for disk file) */
+    if (test->protocol->id == Pudp && test->zerocopy && test->diskfile_name == (char *)0) {
+        test->settings->send_recvmmsg = 1;
+        /* Ensure non-zero burst number of packets when sendmmsg/recvmmsg are used */ 
+	if (test->settings->burst == 0)
+	    test->settings->burst = 1;
+#ifdef UIO_MAXIOV
+	/* Ensure burst size is appropriate for sendmmsg() */
+	else if (test->settings->burst > UIO_MAXIOV)
+	    test->settings->burst = UIO_MAXIOV;
+#endif /* UIO_MAXIOV */
+    }
+#endif /* HAVE_SENDMMSG */
+    return;
+}

src/iperf_locale.c

@@ -211,6 +211,10 @@ const char usage_longstr[] = "Usage: iperf3 [-s|-c host] [options]\n"
                            "  -L, --flowlabel N         set the IPv6 flow label (only supported on Linux)\n"
 #endif /* HAVE_FLOWLABEL */
                            "  -Z, --zerocopy            use a 'zero copy' method of sending data\n"
+#if defined(HAVE_SENDMMSG)
+                           "                            UDP: use `sendmmsg` to send `-b` bust number of messages;\n"
+                           "                                 (not supported when reading from a file).\n"
+#endif /* HAVE_SENDMMSG */
 #if defined(HAVE_MSG_TRUNC)
                            "  --skip-rx-copy            ignore received messages using MSG_TRUNC option\n"
 #endif /* HAVE_MSG_TRUNC */