filehash: implement per-bucket semaphores

thpchallenge-fio from mmtests [1] is frequently used for testing Linux kernel
memory fragmentation [2]. thchallenge-fio writes a large number of 64K files
inefficiently. On a system with 128GiB of RAM, it could create up to
100K files. A typical fio config looks like this

[global]
direct=0
ioengine=sync
blocksize=4096
invalidate=0
fallocate=none
create_on_open=1

[writer]
nrfiles=98200
filesize=65536
readwrite=write
numjobs=16

While testing this on a system with 128GiB of RAM and four NVMe SSDs on
RAID0 that can achieve up to 25GB/s and at least 2M IOPS, it was
observed that for the initial ~20 minutes, the test was running
painfully slow at only ~2MiB/s with 16 fio threads.

After some analysis, it was noticed that during this period, fio spends
most of its time in looking up and adding the files from/to the file
hash table, as shown in the fio report below

    12.04%  [.] __lookup_file_hash

           fio                                             -      -
            |
             --9.94%--__lookup_file_hash
                       |
                        --7.17%--lookup_file_hash
                                  generic_open_file
                                  td_io_open_file
                                  get_io_u
                                  thread_main
                                  run_threads
                                  fio_backend
                                  main
     5.96%  [.] strcmp@plt
           fio                                             -      -
            |
             --5.16%--__lookup_file_hash

    56.38%  [.] __strcmp_evex

           libc.so.6                                       -      -
            |
            |--47.01%--__lookup_file_hash
            |          |
            |          |--35.25%--lookup_file_hash
            |          |          generic_open_file
            |          |          td_io_open_file
            |          |          get_io_u
            |          |          thread_main
            |          |          run_threads
            |          |          fio_backend
            |          |          main
            |          |
            |           --11.76%--add_file_hash
            |                     generic_open_file
            |                     td_io_open_file
            |                     get_io_u
            |                     thread_main
            |                     run_threads
            |                     fio_backend
            |                     main
            |
             --9.31%--__strcmp_evex
                       |
                        --5.31%--add_file_hash
                                  generic_open_file
                                  td_io_open_file
                                  get_io_u
                                  thread_main
                                  run_threads
                                  fio_backend
                                  main

When the hash table is small, the threads spend a lot of time iterating
through the files and comparing the filename. Increasing the size of the
hash table helped significantly improve the performance by 20x, where now
it only takes 1 minute or less. However that comes at the expense of more
memory.

Luckily, because the hash table has a single lock, it was observed that
giving each bucket its own lock helped achieve the same performance
improvment. Since this could potentially help more with multi-threading
scenarios, in this patch, we consider this approach.

Thus this patch implements a fio_sem per bucket. The filename is used to
calculate which semaphore needs to be locked/unlocked.

There is a global filename_list that also originally used the hash_lock
for protection. We assign a random bucket with flist_bucket_name.

[1] https://github.com/gormanm/mmtests/blob/master/configs/config-workload-thpchallenge-fio
[2] https://lwn.net/Articles/770235/

Author: karim-manaouil

filehash.c

@@ -14,39 +14,57 @@
 #define BLOOM_SIZE	16*1024*1024
 
 static unsigned int file_hash_size = HASH_BUCKETS * sizeof(struct flist_head);
+static unsigned int sem_hash_size = HASH_BUCKETS * sizeof(struct fio_sem *);
 
 static struct flist_head *file_hash;
-static struct fio_sem *hash_lock;
+static struct fio_sem **hash_locks;
 static struct bloom *file_bloom;
 
 static unsigned short hash(const char *name)
 {
 	return jhash(name, strlen(name), 0) & HASH_MASK;
 }
 
-void fio_file_hash_lock(void)
+static unsigned int file_bucket(const char *file_name, struct flist_head *bucket)
 {
-	if (hash_lock)
-		fio_sem_down(hash_lock);
+	struct flist_head *head;
+
+	head = &file_hash[hash(file_name)];
+	if (bucket)
+		bucket = head;
+
+	return head - file_hash;
+}
+
+void fio_file_hash_lock(const char *file_name)
+{
+	unsigned int buckidx = file_bucket(file_name, NULL);
+
+	if (hash_locks)
+		fio_sem_down(hash_locks[buckidx]);
 }
 
-void fio_file_hash_unlock(void)
+void fio_file_hash_unlock(const char *file_name)
 {
-	if (hash_lock)
-		fio_sem_up(hash_lock);
+	unsigned int buckidx = file_bucket(file_name, NULL);
+
+	if (hash_locks)
+		fio_sem_up(hash_locks[buckidx]);
 }
 
 void remove_file_hash(struct fio_file *f)
 {
-	fio_sem_down(hash_lock);
+	unsigned long buckidx = file_bucket(f->file_name, NULL);
+
+	fio_sem_down(hash_locks[buckidx]);
 
 	if (fio_file_hashed(f)) {
 		assert(!flist_empty(&f->hash_list));
 		flist_del_init(&f->hash_list);
 		fio_file_clear_hashed(f);
 	}
 
-	fio_sem_up(hash_lock);
+	fio_sem_up(hash_locks[buckidx]);
 }
 
 static struct fio_file *__lookup_file_hash(const char *name)
@@ -71,9 +89,9 @@ struct fio_file *lookup_file_hash(const char *name)
 {
 	struct fio_file *f;
 
-	fio_sem_down(hash_lock);
+	fio_file_hash_lock(name);
 	f = __lookup_file_hash(name);
-	fio_sem_up(hash_lock);
+	fio_file_hash_unlock(name);
 	return f;
 }
 
@@ -86,15 +104,15 @@ struct fio_file *add_file_hash(struct fio_file *f)
 
 	INIT_FLIST_HEAD(&f->hash_list);
 
-	fio_sem_down(hash_lock);
+	fio_file_hash_lock(f->file_name);
 
 	alias = __lookup_file_hash(f->file_name);
 	if (!alias) {
 		fio_file_set_hashed(f);
 		flist_add_tail(&f->hash_list, &file_hash[hash(f->file_name)]);
 	}
 
-	fio_sem_up(hash_lock);
+	fio_file_hash_unlock(f->file_name);
 	return alias;
 }
 
@@ -107,18 +125,19 @@ void file_hash_exit(void)
 {
 	unsigned int i, has_entries = 0;
 
-	fio_sem_down(hash_lock);
-	for (i = 0; i < HASH_BUCKETS; i++)
+	for (i = 0; i < HASH_BUCKETS; i++) {
+		fio_sem_down(hash_locks[i]);
 		has_entries += !flist_empty(&file_hash[i]);
-	fio_sem_up(hash_lock);
+		fio_sem_up(hash_locks[i]);
+		fio_sem_remove(hash_locks[i]);
+	}
 
 	if (has_entries)
 		log_err("fio: file hash not empty on exit\n");
 
 	sfree(file_hash);
 	file_hash = NULL;
-	fio_sem_remove(hash_lock);
-	hash_lock = NULL;
+	hash_locks = NULL;
 	bloom_free(file_bloom);
 	file_bloom = NULL;
 }
@@ -128,10 +147,12 @@ void file_hash_init(void)
 	unsigned int i;
 
 	file_hash = smalloc(file_hash_size);
+	hash_locks = smalloc(sem_hash_size);
 
-	for (i = 0; i < HASH_BUCKETS; i++)
+	for (i = 0; i < HASH_BUCKETS; i++) {
 		INIT_FLIST_HEAD(&file_hash[i]);
+		hash_locks[i] = fio_sem_init(FIO_SEM_UNLOCKED);
+	}
 
-	hash_lock = fio_sem_init(FIO_SEM_UNLOCKED);
 	file_bloom = bloom_new(BLOOM_SIZE);
 }

filehash.h

@@ -8,8 +8,8 @@ extern void file_hash_exit(void);
 extern struct fio_file *lookup_file_hash(const char *);
 extern struct fio_file *add_file_hash(struct fio_file *);
 extern void remove_file_hash(struct fio_file *);
-extern void fio_file_hash_lock(void);
-extern void fio_file_hash_unlock(void);
+extern void fio_file_hash_lock(const char *);
+extern void fio_file_hash_unlock(const char *);
 extern bool file_bloom_exists(const char *, bool);
 
 #endif

filesetup.c

@@ -21,6 +21,8 @@
 #include <linux/falloc.h>
 #endif
 
+static const char *flist_bucket_name = "flist_bucket";
+
 static FLIST_HEAD(filename_list);
 
 /*
@@ -1701,9 +1703,9 @@ static bool is_already_allocated(const char *fname)
 {
 	bool ret;
 
-	fio_file_hash_lock();
+	fio_file_hash_lock(fname);
 	ret = __is_already_allocated(fname, false);
-	fio_file_hash_unlock();
+	fio_file_hash_unlock(fname);
 
 	return ret;
 }
@@ -1715,12 +1717,12 @@ static void set_already_allocated(const char *fname)
 	fn = malloc(sizeof(struct file_name));
 	fn->filename = strdup(fname);
 
-	fio_file_hash_lock();
+	fio_file_hash_lock(flist_bucket_name);
 	if (!__is_already_allocated(fname, true)) {
 		flist_add_tail(&fn->list, &filename_list);
 		fn = NULL;
 	}
-	fio_file_hash_unlock();
+	fio_file_hash_unlock(flist_bucket_name);
 
 	if (fn) {
 		free(fn->filename);
@@ -1736,15 +1738,15 @@ static void free_already_allocated(void)
 	if (flist_empty(&filename_list))
 		return;
 
-	fio_file_hash_lock();
+	fio_file_hash_lock(flist_bucket_name);
 	flist_for_each_safe(entry, tmp, &filename_list) {
 		fn = flist_entry(entry, struct file_name, list);
 		free(fn->filename);
 		flist_del(&fn->list);
 		free(fn);
 	}
 
-	fio_file_hash_unlock();
+	fio_file_hash_unlock(flist_bucket_name);
 }
 
 static struct fio_file *alloc_new_file(struct thread_data *td)