Implement FUSE passthrough (cvmfs#4006)

andrey-utkin · vvolkl · christge-sft · commit ed242b3cf3ea · 2026-02-03T15:12:36.000+01:00
Passthrough is a recent FUSE feature (kernel 6.17.0+, libfuse 3.17+) enabling bypass of userspace on all the read requests. When the FUSE daemon provides a "backing file descriptor" to FUSE on "open" request, the kernel reads directly from it without calling into the FUSE daemon. This is a win for performance. The test supplied demonstrates measurable speedup: ``` + dd status=progress bs=512 if=/cvmfs/test.repo.dumbdumb/big.bin of=/dev/null 10485760 bytes (10 MB, 10 MiB) copied, 3.35429 s, 3.1 MB/s + dd status=progress bs=512 if=/cvmfs/test.repo.passthru/big.bin of=/dev/null 10485760 bytes (10 MB, 10 MiB) copied, 0.073326 s, 143 MB/s ``` ``` + pushd /cvmfs/test.repo.dumbdumb + parallel -N0 'sha1sum --check --status CHECKSUMS' ::: {1..100} real 0m3.440s user 0m4.782s sys 0m3.345s + pushd /cvmfs/test.repo.passthru + parallel -N0 'sha1sum --check --status CHECKSUMS' ::: {1..100} real 0m1.221s user 0m3.904s sys 0m1.709s ``` Enabling new functionality requires * building with sufficiently recent libfuse to have FUSE_CAP_PASSTHROUGH * running with similarly recent enough libfuse * running on Linux kernel 6.17.0 or newer * specifying mount option fuse_passthrough (or fuse_passthru) Related documentation update: cvmfs/doc-cvmfs#254 --------- Co-authored-by: Valentin Volkl <valentin.volkl@cern.ch>
diff --git a/cvmfs/cvmfs.cc b/cvmfs/cvmfs.cc
@@ -130,6 +130,14 @@ FuseRemounter *fuse_remounter_ = NULL;
 InodeGenerationInfo inode_generation_info_;
 #endif  // __TEST_CVMFS_MOCKFUSE
 
+#ifdef FUSE_CAP_PASSTHROUGH
+typedef struct fuse_passthru_ctx {
+  int backing_id;
+  int refcount;
+} fuse_passthru_ctx_t;
+static std::unordered_map<fuse_ino_t, fuse_passthru_ctx_t> *fuse_passthru_tracker = NULL;
+pthread_mutex_t fuse_passthru_tracker_lock = PTHREAD_MUTEX_INITIALIZER;
+#endif
 
 /**
  * For cvmfs_opendir / cvmfs_readdir
@@ -1317,6 +1325,44 @@ static void cvmfs_open(fuse_req_t req, fuse_ino_t ino,
                fd);
       fi->fh = fd;
       FillOpenFlags(open_directives, fi);
+#ifdef FUSE_CAP_PASSTHROUGH
+      if (loader_exports_->fuse_passthrough) {
+        if(!dirent.IsChunkedFile()) {
+          /* "Currently there should be only one backing id per node / backing file."
+           * So says libfuse documentation on fuse_passthrough_open().
+           * So we reuse and refcount backing id based on inode.
+           * Passthrough can be used with libfuse methods open, opendir, create,
+           * but since CVMFS is read-only and has synthesizes its directories,
+           * we only need to handle it in `open`. */
+          int backing_id;
+          pthread_mutex_lock(&fuse_passthru_tracker_lock);
+          auto iter = fuse_passthru_tracker->find(ino);
+          if (iter == fuse_passthru_tracker->end()) {
+            auto pair_with_iterator = fuse_passthru_tracker->emplace(ino, fuse_passthru_ctx_t());
+            assert(pair_with_iterator.second == true);
+            iter = pair_with_iterator.first;
+            fuse_passthru_ctx_t &entry = iter->second;
+
+            backing_id = fuse_passthrough_open(req, fd);
+            assert(backing_id != 0);
+            entry.backing_id = backing_id;
+            entry.refcount++;
+          } else {
+            fuse_passthru_ctx_t &entry = iter->second;
+            assert(entry.refcount > 0);
+            backing_id = entry.backing_id;
+            entry.refcount++;
+          }
+          pthread_mutex_unlock(&fuse_passthru_tracker_lock);
+
+          fi->backing_id = backing_id;
+
+          /* according to libfuse example/passthrough_hp.cc:
+           * "open in passthrough mode must drop old page cache" */
+          fi->keep_cache = false;
+        }
+      }
+#endif
       fuse_reply_open(req, fi);
       return;
     } else {
@@ -1615,6 +1661,30 @@ static void cvmfs_release(fuse_req_t req, fuse_ino_t ino,
     if (file_system_->cache_mgr()->Close(abs_fd) == 0) {
       perf::Dec(file_system_->no_open_files());
     }
+#ifdef FUSE_CAP_PASSTHROUGH
+    if (loader_exports_->fuse_passthrough) {
+
+      if (fi->backing_id != 0) {
+        int ret;
+        pthread_mutex_lock(&fuse_passthru_tracker_lock);
+        auto iter = fuse_passthru_tracker->find(ino);
+        assert(iter != fuse_passthru_tracker->end());
+        fuse_passthru_ctx_t &entry = iter->second;
+        assert(entry.refcount > 0);
+        assert(entry.backing_id == fi->backing_id);
+        entry.refcount--;
+        if (entry.refcount == 0) {
+          ret = fuse_passthrough_close(req, fi->backing_id);
+          if (ret < 0) {
+            LogCvmfs(kLogCvmfs, kLogDebug, "fuse_passthrough_close(fd=%ld) failed: %d", fd, ret);
+            assert(false);
+          }
+          fuse_passthru_tracker->erase(iter);
+        }
+        pthread_mutex_unlock(&fuse_passthru_tracker_lock);
+      }
+    }
+#endif
   }
   fuse_reply_err(req, 0);
 }
@@ -2088,11 +2158,46 @@ static void cvmfs_init(void *userdata, struct fuse_conn_info *conn) {
         FUSE_VERSION);
   }
 #endif
+
+#ifdef FUSE_CAP_PASSTHROUGH
+  if (conn->capable & FUSE_CAP_PASSTHROUGH) {
+    if (loader_exports_->fuse_passthrough) {
+      conn->want |= FUSE_CAP_PASSTHROUGH;
+      /* "Passthrough and writeback cache are conflicting modes"
+       * libfuse example/passthrough_hp.cc says,
+       * but we don't use writeback cache mode in CVMFS. */
+      pthread_mutex_lock(&fuse_passthru_tracker_lock);
+      assert(!fuse_passthru_tracker);
+      fuse_passthru_tracker = new std::unordered_map<fuse_ino_t,
+                                                     fuse_passthru_ctx_t>();
+      pthread_mutex_unlock(&fuse_passthru_tracker_lock);
+      LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslogWarn,
+               "FUSE: Passthrough enabled.");
+    } else {
+      LogCvmfs(kLogCvmfs, kLogDebug,
+               "FUSE: Passthrough enabled in build, available at runtime, but "
+               "not enabled by the config option.");
+    }
+  } else {
+    LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslogWarn,
+             "FUSE: Passthrough enabled in build but unavailable at runtime.");
+  }
+#else
+  LogCvmfs(kLogCvmfs, kLogDebug | kLogSyslogWarn,
+           "FUSE: Passthrough disabled in this build.");
+#endif
 }
 
 static void cvmfs_destroy(void *unused __attribute__((unused))) {
   // The debug log is already closed at this point
   LogCvmfs(kLogCvmfs, kLogDebug, "cvmfs_destroy");
+#ifdef FUSE_CAP_PASSTHROUGH
+  pthread_mutex_lock(&fuse_passthru_tracker_lock);
+  assert(fuse_passthru_tracker);
+  delete fuse_passthru_tracker;
+  fuse_passthru_tracker = NULL;
+  pthread_mutex_unlock(&fuse_passthru_tracker_lock);
+#endif
 }
 
 /**
diff --git a/cvmfs/loader.cc b/cvmfs/loader.cc
@@ -65,6 +65,7 @@ struct CvmfsOptions {
   int simple_options_parsing;
   int foreground;
   int fuse_debug;
+  int fuse_passthrough;
 
   // Ignored options
   int ign_netdev;
@@ -98,6 +99,8 @@ static struct fuse_opt cvmfs_array_opts[] = {
     CVMFS_SWITCH("simple_options_parsing", simple_options_parsing),
     CVMFS_SWITCH("foreground", foreground),
     CVMFS_SWITCH("fuse_debug", fuse_debug),
+    CVMFS_SWITCH("fuse_passthrough", fuse_passthrough),
+    CVMFS_SWITCH("fuse_passthru", fuse_passthrough),
 
     // Ignore these options
     CVMFS_SWITCH("_netdev", ign_netdev),
@@ -142,6 +145,7 @@ bool premounted_ = false;
 bool premount_fuse_ = true;
 bool disable_watchdog_ = false;
 bool simple_options_parsing_ = false;
+bool fuse_passthrough_ = false;
 void *library_handle_;
 Fence *fence_reload_;
 CvmfsExports *cvmfs_exports_;
@@ -170,8 +174,11 @@ static void Usage(const string &exename) {
                             "before mounting (required for autofs)\n"
     "  -o parse             Parse and print cvmfs parameters\n"
     "  -o cvmfs_suid        Enable suid mode\n"
+    "  -o debug             Enable debug to CVMFS_DEBUGLOG\n"
     "  -o disable_watchdog  Do not spawn a post mortem crash handler\n"
     "  -o foreground        Run in foreground\n"
+    "  -o fuse_passthrough  Enables FUSE passthrough (read requests bypass userspace, improves performance)\n"
+    "  -o fuse_passthru     Alias for fuse_passthrough\n"
     "  -o libfuse=[2,3]     Enforce a certain libfuse version\n"
     "Fuse mount options:\n"
     "  -o allow_other       allow access to other users\n"
@@ -420,6 +427,7 @@ static fuse_args *ParseCmdLine(int argc, char *argv[]) {
   if (cvmfs_options.fuse_debug) {
     fuse_opt_add_arg(mount_options, "-d");
   }
+  fuse_passthrough_ = cvmfs_options.fuse_passthrough;
 
   return mount_options;
 }
@@ -793,6 +801,12 @@ int FuseMain(int argc, char *argv[]) {
   loader_exports_->device_id = "0:0";  // initially unknown, set after mount
   loader_exports_->disable_watchdog = disable_watchdog_;
   loader_exports_->simple_options_parsing = simple_options_parsing_;
+  loader_exports_->fuse_passthrough = fuse_passthrough_;
+  if (options_manager->GetValue("CVMFS_FUSE_PASSTHROUGH", &parameter)) {
+    // CVMFS_FUSE_PASSTHROUGH set to on in configs enables the feature.
+    // Presence of mount option can also enable the feature (but not disable it).
+    loader_exports_->fuse_passthrough |= options_manager->IsOn(parameter);
+  }
   if (config_files_)
     loader_exports_->config_files = *config_files_;
   else
diff --git a/cvmfs/loader.h b/cvmfs/loader.h
@@ -162,7 +162,9 @@ struct LoaderExports {
       , foreground(false)
       , disable_watchdog(false)
       , simple_options_parsing(false)
-      , fuse_channel_or_session(NULL) { }
+      , fuse_channel_or_session(NULL)
+      , fuse_passthrough(false)
+  { }
 
   ~LoaderExports() {
     for (unsigned i = 0; i < history.size(); ++i)
@@ -194,6 +196,8 @@ struct LoaderExports {
   // in order to work with both libfuse2 and libfuse3
   void **fuse_channel_or_session;
 
+  bool fuse_passthrough;
+
   // Linux only, stores the major:minor internal mountpoint identifier
   // The identifier is read just after mount from /proc/self/mountinfo
   // If it cannot be determined (e.g. on macOS), device_id is "0:0".
diff --git a/test/src/108-fuse-passthru/main b/test/src/108-fuse-passthru/main
@@ -0,0 +1,18 @@
+#!/bin/sh
+cvmfs_test_name="fuse-passthru"
+cvmfs_test_autofs_on_startup=false
+cvmfs_test_suites=""
+
+cleanup() {
+  true
+}
+
+cvmfs_run_test() {
+  this_script_dir=$TEST_ROOT/src/108-fuse-passthru
+  # We want strict mode here, to catch any failure.
+  # But turning on errexit (set -e) inside a function has no effect.
+  # So we exec our script to keep its error handling mode independent from test/run.sh.
+  exe="$this_script_dir"/run_test
+  echo Launching "$exe"
+  exec "$exe" "$@"
+}
diff --git a/test/src/108-fuse-passthru/run_test b/test/src/108-fuse-passthru/run_test
@@ -0,0 +1,125 @@
+#!/bin/bash
+set -euo pipefail
+set -x
+
+this_script_dir=$(dirname "$(readlink -f "$0")")
+if ! [[ -v TEST_ROOT ]]; then
+    TEST_ROOT=$(readlink -f "$this_script_dir/../..")
+fi
+build_dir=$(dirname "$TEST_ROOT")
+
+mkdir -p /var/{spool,run}/cvmfs/
+[[ -d /etc/cvmfs/default.d ]]
+cat > /etc/cvmfs/default.d/99-test.conf <<EOF
+CVMFS_USER=$(whoami)
+CVMFS_HTTP_PROXY=DIRECT
+CVMFS_AUTO_UPDATE=yes
+CVMFS_USE_FILE_CHUNKING=false
+CVMFS_QUOTA_LIMIT=-1
+EOF
+export CVMFS_TEST_USER=$(whoami)
+
+rsyslogd &
+systemctl enable --now httpd
+
+rm -rf /var/spool/cvmfs/* # just in case
+cp -a /etc/fstab{,.orig}
+> /etc/fstab
+
+create_repo() {
+	MOUNT_OPTS=$1
+	EXTRA_CONFIG=$2
+	export CVMFS_USE_FILE_CHUNKING=false
+	mv /etc/fstab{,.bkp}
+	cvmfs_server mkfs -o "$CVMFS_TEST_USER" "$CVMFS_TEST_REPO"
+	export CVMFS_TEST_MOUNTPOINT=/cvmfs/$CVMFS_TEST_REPO
+	cat "$CVMFS_TEST_MOUNTPOINT"/new_repository
+
+	pgrep cvmfs2
+	umount /cvmfs/"$CVMFS_TEST_REPO"
+	umount /var/spool/cvmfs/"$CVMFS_TEST_REPO"/rdonly
+	sed -i /etc/fstab -e "s/,noauto /,simple_options_parsing,disable_watchdog,noauto,debug,$MOUNT_OPTS /"
+	mv /etc/fstab{,.new}
+	cat /etc/fstab.new /etc/fstab.bkp > /etc/fstab
+	systemctl daemon-reload
+	cat >> /etc/cvmfs/repositories.d/"$CVMFS_TEST_REPO"/client.conf <<-EOF
+	CVMFS_HTTP_PROXY=DIRECT
+	CVMFS_KCACHE_TIMEOUT=0
+	CVMFS_MAX_TTL_SECS=1
+	CVMFS_AUTO_UPDATE=yes
+	CVMFS_DEBUGLOG=/var/log/$CVMFS_TEST_REPO.cvmfs-debug.log
+	CVMFS_USYSLOG=/var/log/$CVMFS_TEST_REPO.cvmfs-usyslog.log
+	$EXTRA_CONFIG
+	EOF
+	cat >> /etc/cvmfs/repositories.d/"$CVMFS_TEST_REPO"/server.conf <<-EOF
+	CVMFS_USE_FILE_CHUNKING=false
+	EOF
+
+	mount  /var/spool/cvmfs/"$CVMFS_TEST_REPO"/rdonly
+	mount  /cvmfs/"$CVMFS_TEST_REPO"
+
+	cvmfs_server transaction "$CVMFS_TEST_REPO"
+	echo HELLO > "$CVMFS_TEST_MOUNTPOINT"/hello.txt
+	time dd status=progress bs=1M count=10 if=/dev/urandom of="$CVMFS_TEST_MOUNTPOINT"/big.bin
+	pushd "$CVMFS_TEST_MOUNTPOINT"
+	sha1sum * > CHECKSUMS
+	popd
+
+	time cvmfs_server publish "$CVMFS_TEST_REPO"
+	mount | grep "$CVMFS_TEST_REPO"
+	cat "$CVMFS_TEST_MOUNTPOINT"/hello.txt
+	grep HELLO "$CVMFS_TEST_MOUNTPOINT"/hello.txt
+
+	if ! ; then
+		umount /cvmfs/"$CVMFS_TEST_REPO"
+		umount /var/spool/cvmfs/"$CVMFS_TEST_REPO"/rdonly
+		nohup valgrind \
+			--tool=memcheck \
+			--log-file=/var/log/"$CVMFS_TEST_REPO".valgrind.log \
+			--enable-debuginfod=yes \
+			--leak-check=full \
+			--show-leak-kinds=all \
+			--track-origins=yes \
+			--trace-children=yes \
+			--track-fds=yes \
+			--show-error-list=all \
+			--read-var-info=yes \
+			--gen-suppressions=all \
+			--suppressions=<(grep -v '^[=-]' "$build_dir"/valgrind.supp) \
+			\
+			/usr/bin/cvmfs2 -f -o \
+			allow_other,fsname="$CVMFS_TEST_REPO",config=/etc/cvmfs/repositories.d/"$CVMFS_TEST_REPO"/client.conf:/var/spool/cvmfs/"$CVMFS_TEST_REPO"/client.local,disable_watchdog,simple_options_parsing,grab_mountpoint,uid=0,gid=0"$MOUNT_OPTS" \
+			"$CVMFS_TEST_REPO" /var/spool/cvmfs/"$CVMFS_TEST_REPO"/rdonly \
+			&
+		while ! grep /var/spool/cvmfs/"$CVMFS_TEST_REPO"/rdonly /proc/mounts; do
+			sleep 1
+		done
+		mount /cvmfs/"$CVMFS_TEST_REPO"
+	fi
+
+	touch /tmp/"$CVMFS_TEST_REPO".big.bin.read.begin
+	time dd status=progress bs=512 if="$CVMFS_TEST_MOUNTPOINT"/big.bin of=/dev/null
+	touch /tmp/"$CVMFS_TEST_REPO".big.bin.read.end
+	DURATION=$(echo -e "scale=10\n $(date +%s.%N --reference=/tmp/"$CVMFS_TEST_REPO".big.bin.read.end) - $(date +%s.%N --reference=/tmp/"$CVMFS_TEST_REPO".big.bin.read.begin)\n" | bc -q )
+	echo "$DURATION" > /tmp/"$CVMFS_TEST_REPO".big.bin.read.duration
+
+	pushd "$CVMFS_TEST_MOUNTPOINT"
+	time parallel -N0 "sha1sum --check --status CHECKSUMS" ::: {1..100}
+	popd
+	umount /cvmfs/"$CVMFS_TEST_REPO"
+	umount /var/spool/cvmfs/"$CVMFS_TEST_REPO"/rdonly
+}
+
+export CVMFS_TEST_REPO=test.repo.dumbdumb
+create_repo '' ''
+grep -F 'FUSE: Passthrough enabled in build, available at runtime, but not enabled by the config option.' /var/log/test.repo.dumbdumb.cvmfs-debug.log
+
+export CVMFS_TEST_REPO=test.repo.passthru
+create_repo ,fuse_passthrough ''
+grep -F 'FUSE: Passthrough enabled.' /var/log/test.repo.passthru.cvmfs-debug.log
+
+export CVMFS_TEST_REPO=test.repo.with-var
+create_repo '' 'CVMFS_FUSE_PASSTHROUGH=on'
+grep -F 'FUSE: Passthrough enabled.' /var/log/test.repo.with-var.cvmfs-debug.log
+
+head /tmp/test.repo.{passthru,dumbdumb}.big.bin.read.duration
