Improve concurrent sequential access speed by per-file file pointer.

This commit extends the caching of cluster information introduced by
commit 6c332e1 so that multiple open file handles accessing the same
node each have their own cache. That prevents dramatic slowdowns when
multiple processes are reading from a large file.

* Replace the fptr_index and fptr_cluster members of struct exfat_node
  introduced by 6c332e1 with a list of exfat_fptr structs, each of
  which holds one index and cluster.

* In fuse_exfat_open, add a new exfat_fptr to the exfat_node, and
  store a pointer to both in the fh member of the fuse_file_info.

* In fuse_exfat_read and fuse_exfat_write, pass the exfat_fptr from
  the fh through to exfat_advance_cluster, so that multiple file
  handles open on the same file can independently track their most
  recently used cluster.

  For all other uses of exfat_advance_cluster, we continue to user the
  "shared" exfat_fptr stored in the node.

* Adjust grow_file and shrink_file to detect when multiple exfat_fptr
  instances are stored in the node and update them all as needed.

Author: tmhannes

fuse/main.c

@@ -44,15 +44,69 @@
 
 struct exfat ef;
 
-static struct exfat_node* get_node(const struct fuse_file_info* fi)
+struct exfat_fh
 {
-	return (struct exfat_node*) (size_t) fi->fh;
+	struct exfat_node *node;
+	struct exfat_fptr fptr;
+};
+
+static struct exfat_fh *get_fh(const struct fuse_file_info *fi) {
+	return (struct exfat_fh *)(size_t)fi->fh;
 }
 
-static void set_node(struct fuse_file_info* fi, struct exfat_node* node)
-{
-	fi->fh = (uint64_t) (size_t) node;
+static struct exfat_node *get_node(const struct fuse_file_info *fi) {
+	return get_fh(fi)->node;
+}
+
+static struct exfat_fptr *get_fptr(const struct fuse_file_info *fi) {
+	return &get_fh(fi)->fptr;
+}
+
+static int add_fh(struct fuse_file_info *fi, struct exfat_node *node) {
+	/* allocate and init new file handle */
+	struct exfat_fh *fh = malloc(sizeof(struct exfat_fh));
+	if (fh == NULL) {
+		exfat_error("failed to allocate file handle");
+		return -ENOMEM;
+	}
+	memset(fh, 0, sizeof(struct exfat_fh));
+	fh->node = node;
+	fh->fptr.cluster = node->start_cluster;
+
+	/* add the file handle's fptr to the node */
+	struct exfat_fptr *p = &node->fptr;
+	int count = 1;
+	while(p->next != NULL) {
+		p = p->next;
+		++count;
+	}
+	p->next = &fh->fptr;
+	exfat_debug("added file handle %d to node", count);
+
+	/* hand the file handle to fuse */
+	fi->fh = (uint64_t)(size_t)fh;
 	fi->keep_cache = 1;
+	return 0;
+}
+
+static void remove_fh(struct fuse_file_info *fi) {
+	struct exfat_fh *fh = get_fh(fi);
+
+	struct exfat_fptr *p = &fh->node->fptr;
+	while (p->next != &fh->fptr) {
+		p = p->next;
+		if (p == NULL)
+			exfat_bug("freeing a file handle that wasn't listed in the node");
+	}
+	p->next = p->next->next;
+
+	exfat_debug("removed a file handle from node");
+	if(fh->node->fptr.next == NULL)
+		exfat_debug("no more file handles open on this node");
+
+	free(fh);
+
+	fi->fh = 0;
 }
 
 static int fuse_exfat_getattr(const char* path, struct stat* stbuf)
@@ -152,7 +206,9 @@ static int fuse_exfat_open(const char* path, struct fuse_file_info* fi)
 	rc = exfat_lookup(&ef, &node, path);
 	if (rc != 0)
 		return rc;
-	set_node(fi, node);
+	rc = add_fh(fi, node);
+	if (rc != 0)
+		return rc;
 	return 0;
 }
 
@@ -170,7 +226,9 @@ static int fuse_exfat_create(const char* path, UNUSED mode_t mode,
 	rc = exfat_lookup(&ef, &node, path);
 	if (rc != 0)
 		return rc;
-	set_node(fi, node);
+	rc = add_fh(fi, node);
+	if (rc != 0)
+		return rc;
 	return 0;
 }
 
@@ -186,6 +244,7 @@ static int fuse_exfat_release(UNUSED const char* path,
 	exfat_debug("[%s] %s", __func__, path);
  	exfat_flush_node(&ef, get_node(fi));
 	exfat_put_node(&ef, get_node(fi));
+	remove_fh(fi);
 	return 0; /* FUSE ignores this return value */
 }
 
@@ -220,14 +279,14 @@ static int fuse_exfat_read(UNUSED const char* path, char* buffer,
 		size_t size, off_t offset, struct fuse_file_info* fi)
 {
 	exfat_debug("[%s] %s (%zu bytes)", __func__, path, size);
-	return exfat_generic_pread(&ef, get_node(fi), buffer, size, offset);
+	return exfat_generic_pread(&ef, get_node(fi), buffer, size, offset, get_fptr(fi));
 }
 
 static int fuse_exfat_write(UNUSED const char* path, const char* buffer,
 		size_t size, off_t offset, struct fuse_file_info* fi)
 {
 	exfat_debug("[%s] %s (%zu bytes)", __func__, path, size);
-	return exfat_generic_pwrite(&ef, get_node(fi), buffer, size, offset);
+	return exfat_generic_pwrite(&ef, get_node(fi), buffer, size, offset, get_fptr(fi));
 }
 
 static int fuse_exfat_unlink(const char* path)

libexfat/cluster.c

@@ -90,25 +90,33 @@ cluster_t exfat_next_cluster(const struct exfat* ef,
 }
 
 cluster_t exfat_advance_cluster(const struct exfat* ef,
-		struct exfat_node* node, uint32_t count)
+		struct exfat_node* node, uint32_t count,
+		struct exfat_fptr* fptr)
 {
 	uint32_t i;
+	uint32_t *index;
+	cluster_t *cluster;
 
-	if (node->fptr_index > count)
+	if(fptr == NULL)
+		/* Fallback for cases where don't have a per-file-handle file
+		 * pointer. */
+		fptr = &node->fptr;
+
+	if (fptr->index > count)
 	{
-		node->fptr_index = 0;
-		node->fptr_cluster = node->start_cluster;
+		fptr->index = 0;
+		fptr->cluster = node->start_cluster;
 	}
 
-	for (i = node->fptr_index; i < count; i++)
+	for (i = fptr->index; i < count; i++)
 	{
-		node->fptr_cluster = exfat_next_cluster(ef, node, node->fptr_cluster);
-		if (CLUSTER_INVALID(*ef->sb, node->fptr_cluster))
+		fptr->cluster = exfat_next_cluster(ef, node, fptr->cluster);
+		if (CLUSTER_INVALID(*ef->sb, fptr->cluster))
 			break; /* the caller should handle this and print appropriate 
 			          error message */
 	}
-	node->fptr_index = count;
-	return node->fptr_cluster;
+	fptr->index = count;
+	return fptr->cluster;
 }
 
 static cluster_t find_bit_and_set(bitmap_t* bitmap, size_t start, size_t end)
@@ -249,7 +257,7 @@ static int grow_file(struct exfat* ef, struct exfat_node* node,
 	if (node->start_cluster != EXFAT_CLUSTER_FREE)
 	{
 		/* get the last cluster of the file */
-		previous = exfat_advance_cluster(ef, node, current - 1);
+		previous = exfat_advance_cluster(ef, node, current - 1, NULL);
 		if (CLUSTER_INVALID(*ef->sb, previous))
 		{
 			exfat_error("invalid cluster 0x%x while growing", previous);
@@ -258,14 +266,17 @@ static int grow_file(struct exfat* ef, struct exfat_node* node,
 	}
 	else
 	{
-		if (node->fptr_index != 0)
-			exfat_bug("non-zero pointer index (%u)", node->fptr_index);
 		/* file does not have clusters (i.e. is empty), allocate
 		   the first one for it */
 		previous = allocate_cluster(ef, 0);
 		if (CLUSTER_INVALID(*ef->sb, previous))
 			return -ENOSPC;
-		node->fptr_cluster = node->start_cluster = previous;
+		for(struct exfat_fptr* fptr=&node->fptr; fptr != NULL; fptr=fptr->next) {
+			if (fptr->index != 0)
+				exfat_bug("non-zero pointer index (%u)", fptr->index);
+			fptr->cluster = previous;
+		}
+		node->start_cluster = previous;
 		allocated = 1;
 		/* file consists of only one cluster, so it's contiguous */
 		node->is_contiguous = true;
@@ -318,7 +329,7 @@ static int shrink_file(struct exfat* ef, struct exfat_node* node,
 	if (current > difference)
 	{
 		cluster_t last = exfat_advance_cluster(ef, node,
-				current - difference - 1);
+				current - difference - 1, NULL);
 		if (CLUSTER_INVALID(*ef->sb, last))
 		{
 			exfat_error("invalid cluster 0x%x while shrinking", last);
@@ -335,8 +346,10 @@ static int shrink_file(struct exfat* ef, struct exfat_node* node,
 		node->start_cluster = EXFAT_CLUSTER_FREE;
 		node->is_dirty = true;
 	}
-	node->fptr_index = 0;
-	node->fptr_cluster = node->start_cluster;
+	for(struct exfat_fptr* fptr=&node->fptr; fptr != NULL; fptr=fptr->next) {
+		fptr->index = 0;
+		fptr->cluster = node->start_cluster;
+	}
 
 	/* free remaining clusters */
 	while (difference--)
@@ -379,7 +392,7 @@ static int erase_range(struct exfat* ef, struct exfat_node* node,
 
 	cluster_boundary = (begin | (CLUSTER_SIZE(*ef->sb) - 1)) + 1;
 	cluster = exfat_advance_cluster(ef, node,
-			begin / CLUSTER_SIZE(*ef->sb));
+			begin / CLUSTER_SIZE(*ef->sb), NULL);
 	if (CLUSTER_INVALID(*ef->sb, cluster))
 	{
 		exfat_error("invalid cluster 0x%x while erasing", cluster);

libexfat/exfat.h

@@ -71,6 +71,13 @@
    be corrupted with 32-bit off_t. */
 STATIC_ASSERT(sizeof(off_t) == 8);
 
+struct exfat_fptr
+{
+	uint32_t index;
+	cluster_t cluster;
+	struct exfat_fptr* next;
+};
+
 struct exfat_node
 {
 	struct exfat_node* parent;
@@ -79,8 +86,7 @@ struct exfat_node
 	struct exfat_node* prev;
 
 	int references;
-	uint32_t fptr_index;
-	cluster_t fptr_cluster;
+	struct exfat_fptr fptr;
 	off_t entry_offset;
 	cluster_t start_cluster;
 	uint16_t attrib;
@@ -162,9 +168,9 @@ ssize_t exfat_pread(struct exfat_dev* dev, void* buffer, size_t size,
 ssize_t exfat_pwrite(struct exfat_dev* dev, const void* buffer, size_t size,
 		off_t offset);
 ssize_t exfat_generic_pread(const struct exfat* ef, struct exfat_node* node,
-		void* buffer, size_t size, off_t offset);
+		void* buffer, size_t size, off_t offset, struct exfat_fptr* fh);
 ssize_t exfat_generic_pwrite(struct exfat* ef, struct exfat_node* node,
-		const void* buffer, size_t size, off_t offset);
+		const void* buffer, size_t size, off_t offset, struct exfat_fptr* fh);
 
 int exfat_opendir(struct exfat* ef, struct exfat_node* dir,
 		struct exfat_iterator* it);
@@ -179,7 +185,7 @@ off_t exfat_c2o(const struct exfat* ef, cluster_t cluster);
 cluster_t exfat_next_cluster(const struct exfat* ef,
 		const struct exfat_node* node, cluster_t cluster);
 cluster_t exfat_advance_cluster(const struct exfat* ef,
-		struct exfat_node* node, uint32_t count);
+		struct exfat_node* node, uint32_t count, struct exfat_fptr* fptr);
 int exfat_flush_nodes(struct exfat* ef);
 int exfat_flush(struct exfat* ef);
 int exfat_truncate(struct exfat* ef, struct exfat_node* node, uint64_t size,

libexfat/io.c

@@ -388,7 +388,7 @@ ssize_t exfat_pwrite(struct exfat_dev* dev, const void* buffer, size_t size,
 }
 
 ssize_t exfat_generic_pread(const struct exfat* ef, struct exfat_node* node,
-		void* buffer, size_t size, off_t offset)
+		void* buffer, size_t size, off_t offset, struct exfat_fptr* fptr)
 {
 	uint64_t newsize = offset;
 	cluster_t cluster;
@@ -402,7 +402,7 @@ ssize_t exfat_generic_pread(const struct exfat* ef, struct exfat_node* node,
 	if (size == 0)
 		return 0;
 
-	cluster = exfat_advance_cluster(ef, node, newsize / CLUSTER_SIZE(*ef->sb));
+	cluster = exfat_advance_cluster(ef, node, newsize / CLUSTER_SIZE(*ef->sb), fptr);
 	if (CLUSTER_INVALID(*ef->sb, cluster))
 	{
 		exfat_error("invalid cluster 0x%x while reading", cluster);
@@ -436,7 +436,7 @@ ssize_t exfat_generic_pread(const struct exfat* ef, struct exfat_node* node,
 }
 
 ssize_t exfat_generic_pwrite(struct exfat* ef, struct exfat_node* node,
-		const void* buffer, size_t size, off_t offset)
+		const void* buffer, size_t size, off_t offset, struct exfat_fptr* fptr)
 {
 	uint64_t newsize = offset;
 	int rc;
@@ -461,7 +461,7 @@ ssize_t exfat_generic_pwrite(struct exfat* ef, struct exfat_node* node,
 	if (size == 0)
 		return 0;
 
-	cluster = exfat_advance_cluster(ef, node, newsize / CLUSTER_SIZE(*ef->sb));
+	cluster = exfat_advance_cluster(ef, node, newsize / CLUSTER_SIZE(*ef->sb), fptr);
 	if (CLUSTER_INVALID(*ef->sb, cluster))
 	{
 		exfat_error("invalid cluster 0x%x while writing", cluster);

libexfat/mount.c

@@ -303,7 +303,7 @@ int exfat_mount(struct exfat* ef, const char* spec, const char* options)
 	memset(ef->root, 0, sizeof(struct exfat_node));
 	ef->root->attrib = EXFAT_ATTRIB_DIR;
 	ef->root->start_cluster = le32_to_cpu(ef->sb->rootdir_cluster);
-	ef->root->fptr_cluster = ef->root->start_cluster;
+	ef->root->fptr.cluster = ef->root->start_cluster;
 	ef->root->name[0] = cpu_to_le16('\0');
 	ef->root->size = rootdir_size(ef);
 	if (ef->root->size == 0)

libexfat/node.c

@@ -67,6 +67,9 @@ int exfat_cleanup_node(struct exfat* ef, struct exfat_node* node)
 		exfat_bug("unable to cleanup a node with %d references",
 				node->references);
 
+	if (node->fptr.next != NULL)
+		exfat_bug("unable to cleanup a node with an open file handle");
+
 	if (node->is_unlinked)
 	{
 		/* free all clusters and node structure itself */
@@ -86,7 +89,7 @@ static int read_entries(struct exfat* ef, struct exfat_node* dir,
 		exfat_bug("attempted to read entries from a file");
 
 	size = exfat_generic_pread(ef, dir, entries,
-			sizeof(struct exfat_entry[n]), offset);
+			sizeof(struct exfat_entry[n]), offset, NULL);
 	if (size == (ssize_t) sizeof(struct exfat_entry) * n)
 		return 0; /* success */
 	if (size == 0)
@@ -107,7 +110,7 @@ static int write_entries(struct exfat* ef, struct exfat_node* dir,
 		exfat_bug("attempted to write entries into a file");
 
 	size = exfat_generic_pwrite(ef, dir, entries,
-			sizeof(struct exfat_entry[n]), offset);
+			sizeof(struct exfat_entry[n]), offset, NULL);
 	if (size == (ssize_t) sizeof(struct exfat_entry) * n)
 		return 0; /* success */
 	if (size < 0)
@@ -146,7 +149,7 @@ static void init_node_meta2(struct exfat_node* node,
 {
 	node->size = le64_to_cpu(meta2->size);
 	node->start_cluster = le32_to_cpu(meta2->start_cluster);
-	node->fptr_cluster = node->start_cluster;
+	node->fptr.cluster = node->start_cluster;
 	node->is_contiguous = ((meta2->flags & EXFAT_FLAG_CONTIGUOUS) != 0);
 }
 

libexfat/repair.c

@@ -94,7 +94,7 @@ bool exfat_fix_unknown_entry(struct exfat* ef, struct exfat_node* dir,
 
 	deleted.type &= ~EXFAT_ENTRY_VALID;
 	if (exfat_generic_pwrite(ef, dir, &deleted, sizeof(struct exfat_entry),
-			offset) != sizeof(struct exfat_entry))
+			offset, NULL) != sizeof(struct exfat_entry))
 		return false;
 
 	exfat_errors_fixed++;