Add pthread_barrier event

This is just like the barrier event except it actually uses
pthread_barrier_wait instead of pthread_cond_wait and
pthread_cond_broadcast. This alternative method may cause issues with
killing rt-app, but is still desirable because it exercises different
syscall usage patterns, which are used in real workloads. I have
tried to explain the details of this in tutorial.txt

Implementing this requires adding a `finalize` step to resource
setup, because we need to know the number of threads that refer to
the resource before we can call pthread_barrier_init.

Author: Brendan Jackman

doc/tutorial.txt

@@ -521,7 +521,55 @@ generates the following sequence:
     "unlock" : "SyncPointA" (internal mutex)
 }
 
-* suspend : String. Block the calling thread until another thread wakes it up
+* pthread_barrier: String. Used exactly like barrier, however this actually uses
+pthread_barrier_wait. As discussed above, this is likely to cause problems with
+killing rt-app.
+
+Why would you want this instead of 'barrier'? 'barrier' uses pthread_cond_wait
+and pthread_broadcast. Note that pthread_cond_wait is passed a mutex, and that
+this mutex must be held by the caller. pthread_cond_wait releases this mutex,
+and must re-acquire it before returning. That means that when multiple waiters
+are woken by a pthread_cond_broadcast, all but one of them would have to
+immediately go back to sleep, waiting on the mutex, before returning from
+pthread_cond_wait. Therefore pthread implementations use a different futex
+mechanism for pthread_cond_wait than for pthread_barrier_wait: instead of using
+FUTEX_WAKE to wake all the waiters, FUTEX_CMP_REQUEUE is used to wake _one_of
+the waiters, and _move_ the remaining waiters so that instead of waiting on the
+condition variable itself, they are now waiting on the mutex. When the first
+waiter is woken it will release the mutex before returning from
+pthread_cond_wait, which will wake the second waiter. When the second waiter
+releases the mutex, it will release the third waiter, and so on. This can impact
+scheduler behaviour - instead of waking up multiple threads from a single
+context we now wake one thread from each context:
+
+Using 'pthread_barrier' event:
+
+
+ Thread A                       | Thread B                      | Thread C
+=========================================================================================
+                                |                               | pthread_barrier_wait(&b)
+                                | pthread_barrier_wait(&b)      |
+ pthread_barrier_wait(&b)       |                               |
+                     \--WAKES-->|      # unblocked by A #       |
+                      \--WAKES--------------------------------->|      # unblocked by A #
+
+
+Using 'barrier' event:
+
+
+ Thread A                       | Thread B                      | Thread C
+=========================================================================================
+                                |                               | pthread_cond_wait(&c, &m)
+                                | pthread_cond_wait(&c, &m)     |
+ pthread_cond_broadcast(&c)     |                               |
+                     \--WAKES-->|      # unblocked by A #       |
+                                |                    \--WAKES-->|     # unblocked by B #
+
+
+Therefore, you might want to use the pthread_barrier event even though it can
+cause problems with killing rt-app.
+
+* Suspend : String. Block the calling thread until another thread wakes it up
 with resume. The String can be let empty as it will be filled by workgen with
 the right thread's name before starting the use case.
 

src/rt-app.c

@@ -299,6 +299,10 @@ static int run_event(event_data_t *event, int dry_run,
 		}
 		pthread_mutex_unlock(&(rdata->res.barrier.m_obj));
 		break;
+        case rtapp_pthread_barrier:
+                log_debug("pthread_barrier %s", rdata->name);
+                pthread_barrier_wait(&rdata->res.pthread_barrier.obj);
+                break;
 	case rtapp_sig_and_wait:
 		log_debug("signal and wait %s", rdata->name);
 		pthread_cond_signal(&(rdata->res.cond.obj));

src/rt-app_parse_config.c

@@ -238,6 +238,19 @@ static void init_barrier_resource(rtapp_resource_t *data, const rtapp_options_t
 	pthread_cond_init(&data->res.barrier.c_obj, NULL);
 }
 
+static void finalize_pthread_barrier_resource(rtapp_resource_t *data, const rtapp_options_t *opts)
+{
+	int err;
+	int thread_count = data->res.pthread_barrier.thread_ref_count;
+
+	log_info(PIN3 "Finalize: %s pthread_barrier %d threads", data->name, thread_count);
+
+	err = pthread_barrier_init(&data->res.pthread_barrier.obj, NULL, thread_count);
+	if (err) {
+		perror("pthread_barrier_init");
+	}
+}
+
 static void
 init_resource_data(const char *name, int type, int idx, const rtapp_options_t *opts)
 {
@@ -267,6 +280,9 @@ init_resource_data(const char *name, int type, int idx, const rtapp_options_t *o
 		case rtapp_barrier:
 			init_barrier_resource(data, opts);
 			break;
+		case rtapp_pthread_barrier:
+			/* Init all done in finalize_resources */
+			break;
 		default:
 			break;
 	}
@@ -357,6 +373,21 @@ static int get_resource_index(const char *name, int type, rtapp_options_t *opts)
 	return i;
 }
 
+static int finalize_resources(rtapp_options_t *opts)
+{
+	int i;
+
+	for (i = 0; i <  opts->nresources; i++) {
+		switch (opts->resources[i].type) {
+			case rtapp_pthread_barrier:
+				finalize_pthread_barrier_resource(&opts->resources[i], opts);
+				break;
+			default:
+				break;
+		}
+	}
+}
+
 static char* create_unique_name(char *tmp, int size, const char* ref, long tag)
 {
 	snprintf(tmp, size, "%s%lx", ref, (long)(tag));
@@ -499,21 +530,30 @@ parse_thread_event_data(char *name, struct json_object *obj,
 		return;
 	}
 
-    if (!strncmp(name, "barrier", strlen("barrier"))) {
+	if (!strncmp(name, "barrier", strlen("barrier")) ||
+	    !strncmp(name, "pthread_barrier", strlen("pthread_barrier"))) {
+		int thread_count;
 
 		if (!json_object_is_type(obj, json_type_string))
 			goto unknown_event;
 
-		data->type = rtapp_barrier;
+		if (!strncmp(name, "barrier", strlen("barrier")))
+			data->type = rtapp_barrier;
+		else
+			data->type = rtapp_pthread_barrier;
 
 		ref = json_object_get_string(obj);
-		i = get_resource_index(ref, rtapp_barrier, opts);
+		i = get_resource_index(ref, data->type, opts);
 
 		data->res = i;
 		rdata = &(opts->resources[data->res]);
-		rdata->res.barrier.waiting += 1;
+		if (data->type == rtapp_barrier)
+			thread_count = ++rdata->res.barrier.waiting;
+		else
+			thread_count = ++rdata->res.pthread_barrier.thread_ref_count;
 
-		log_info(PIN2 "type %d target %s [%d] %d users so far", data->type, rdata->name, rdata->index, rdata->res.barrier.waiting);
+		log_info(PIN2 "type %d target %s [%d] %d users so far",
+			 data->type, rdata->name, rdata->index, thread_count);
 		return;
 	}
 
@@ -631,6 +671,7 @@ static char *events[] = {
 	"iorun",
 	"yield",
 	"barrier",
+	"pthread_barrier",
 	NULL
 };
 
@@ -1051,7 +1092,8 @@ get_opts_from_json_object(struct json_object *root, rtapp_options_t *opts)
 	parse_tasks(tasks, opts);
 	json_object_put(tasks);
 	log_info(PFX "Free json objects");
-
+	log_info(PFX "Finalize resources");
+	finalize_resources(opts);
 }
 
 void

src/rt-app_types.h

@@ -72,7 +72,8 @@ typedef enum resource_t
 	rtapp_iorun,
 	rtapp_runtime,
 	rtapp_yield,
-	rtapp_barrier
+	rtapp_barrier,
+	rtapp_pthread_barrier
 } resource_t;
 
 struct _rtapp_mutex {
@@ -99,6 +100,12 @@ struct _rtapp_barrier_like {
 	pthread_cond_t c_obj;
 };
 
+struct _rtapp_pthread_barrier {
+        pthread_barrier_t obj;
+        /* Number of threads that refer to this barrier */
+        int thread_ref_count;
+};
+
 struct _rtapp_signal {
 	pthread_cond_t *target;
 };
@@ -128,6 +135,7 @@ typedef struct _rtapp_resource_t {
 		struct _rtapp_iomem_buf buf;
 		struct _rtapp_iodev dev;
 		struct _rtapp_barrier_like barrier;
+		struct _rtapp_pthread_barrier pthread_barrier;
 	} res;
 	int index;
 	resource_t type;