feat: add --overlay flag for shared compilation caches

src/setup.rs

@@ -150,6 +150,14 @@ pub struct MountOptions {
     /// When not set, requires `user_allow_other` in /etc/fuse.conf on Linux.
     #[arg(long, default_value_t = false)]
     pub fuse_owner_only: bool,
+
+    /// Enable overlay mode. The mount point directory serves as the local
+    /// layer: pre-existing files are visible through the mount, and new
+    /// writes persist there in their original path layout. Reads merge
+    /// local files with remote bucket contents (local takes precedence).
+    /// Implies --advanced-writes. Writes are never pushed to remote.
+    #[arg(long, default_value_t = false)]
+    pub overlay: bool,
 }
 
 /// CLI args for the foreground FUSE/NFS binaries.
@@ -175,6 +183,8 @@ pub struct MountSetup {
     pub max_threads: usize,
     pub metadata_ttl_ms: u64,
     pub fuse_owner_only: bool,
+    /// Kept alive to preserve access to the underlying dir via /proc/self/fd/N.
+    pub _overlay_fd: Option<std::fs::File>,
 }
 
 // ── Tracing + env vars (no threads) ──────────────────────────────────
@@ -280,12 +290,21 @@ pub fn build(source: Source, options: MountOptions, is_nfs: bool) -> MountSetup
         });
     }
 
-    let read_only = options.read_only || hub_client.is_repo();
-    if hub_client.is_repo() && !options.read_only {
+    if options.overlay && options.read_only {
+        panic!(
+            "--overlay with --read-only is pointless: overlay enables local writes, --read-only disables them. Use --read-only alone instead."
+        );
+    }
+
+    let read_only = (options.read_only || hub_client.is_repo()) && !options.overlay;
+    if hub_client.is_repo() && !options.read_only && !options.overlay {
         info!("Repo mounts are always read-only");
     }
 
-    let refresher = hub_client.token_refresher(read_only);
+    // Overlay mode: VFS is read-write (local writes), but remote is read-only
+    // (no write token, no upload config).
+    let remote_read_only = read_only || options.overlay;
+    let refresher = hub_client.token_refresher(remote_read_only);
     let cas_config = build_cas_config(&runtime, &refresher);
 
     // Ensure cache directory exists and is writable (needed for staging even without chunk cache).
@@ -314,14 +333,39 @@ pub fn build(source: Source, options: MountOptions, is_nfs: bool) -> MountSetup
         .expect("Failed to create storage client");
     let cached_client = CachedXetClient::new(raw_client);
     let download_session = FileDownloadSession::from_client(cached_client.clone(), None, xorb_cache);
-    let upload_config = if read_only { None } else { Some(cas_config) };
+    let upload_config = if remote_read_only { None } else { Some(cas_config) };
     let xet_sessions = XetSessions::new(download_session, upload_config, cached_client);
 
-    let advanced_writes = options.advanced_writes || (is_nfs && !read_only);
+    let advanced_writes = options.advanced_writes || options.overlay || (is_nfs && !read_only);
+
+    // In overlay mode, open a fd to the mount point directory BEFORE mounting.
+    // This preserves access to the underlying local files after the mount shadows them.
+    let overlay_fd = if options.overlay {
+        use std::os::unix::io::AsRawFd;
+        // Ensure mount point exists before opening fd
+        std::fs::create_dir_all(&mount_point)
+            .unwrap_or_else(|e| panic!("Failed to create mount point {:?} for overlay: {e}", mount_point));
+        let fd = std::fs::File::open(&mount_point)
+            .unwrap_or_else(|e| panic!("Failed to open mount point {:?} for overlay: {e}", mount_point));
+        let raw_fd = fd.as_raw_fd();
+        #[cfg(target_os = "linux")]
+        let overlay_root = PathBuf::from(format!("/proc/self/fd/{}", raw_fd));
+        #[cfg(not(target_os = "linux"))]
+        let overlay_root = PathBuf::from(format!("/dev/fd/{}", raw_fd));
+        info!("Overlay mode: local dir accessible via {:?}", overlay_root);
+        Some((fd, overlay_root))
+    } else {
+        None
+    };
+
     // Repos need a staging dir for HTTP download cache (open_readonly),
     // even when advanced_writes is disabled.
     let staging_dir = if advanced_writes || hub_client.is_repo() {
-        Some(StagingDir::new(&options.cache_dir))
+        if let Some((_, ref overlay_root)) = overlay_fd {
+            Some(StagingDir::new_overlay(&options.cache_dir, overlay_root.clone()))
+        } else {
+            Some(StagingDir::new(&options.cache_dir))
+        }
     } else {
         None
     };
@@ -356,10 +400,11 @@ pub fn build(source: Source, options: MountOptions, is_nfs: bool) -> MountSetup
         backend_name,
     );
     info!(
-        "Config: advanced_writes={} direct_io={} poll_interval={}s metadata_ttl={}ms \
+        "Config: advanced_writes={} overlay={} direct_io={} poll_interval={}s metadata_ttl={}ms \
          cache_dir={:?} cache_size={} no_disk_cache={} max_threads={} \
          flush_debounce={}ms flush_max_batch={}ms uid={} gid={} filter_os_files={}",
         advanced_writes,
+        options.overlay,
         options.direct_io,
         options.poll_interval_secs,
         options.metadata_ttl_ms,
@@ -384,6 +429,7 @@ pub fn build(source: Source, options: MountOptions, is_nfs: bool) -> MountSetup
         VfsConfig {
             read_only,
             advanced_writes,
+            overlay: options.overlay,
             uid,
             gid,
             poll_interval_secs: options.poll_interval_secs,
@@ -407,6 +453,7 @@ pub fn build(source: Source, options: MountOptions, is_nfs: bool) -> MountSetup
         max_threads: options.max_threads,
         metadata_ttl_ms: options.metadata_ttl_ms,
         fuse_owner_only: options.fuse_owner_only,
+        _overlay_fd: overlay_fd.map(|(fd, _)| fd),
     }
 }
 

src/test_mocks.rs

@@ -443,6 +443,7 @@ impl DownloadStreamOps for MockDownloadStream {
 pub struct TestOpts {
     pub read_only: bool,
     pub advanced_writes: bool,
+    pub overlay: bool,
     pub serve_lookup_from_cache: bool,
     pub metadata_ttl: Duration,
 }
@@ -452,12 +453,21 @@ impl Default for TestOpts {
         Self {
             read_only: false,
             advanced_writes: false,
+            overlay: false,
             serve_lookup_from_cache: false,
             metadata_ttl: Duration::from_secs(1),
         }
     }
 }
 
+/// Return value from `make_test_vfs` when overlay mode is used.
+/// Includes the overlay root path so tests can pre-populate or inspect local files.
+pub struct OverlayTestVfs {
+    pub runtime: tokio::runtime::Runtime,
+    pub vfs: Arc<crate::virtual_fs::VirtualFs>,
+    pub overlay_root: std::path::PathBuf,
+}
+
 /// Build a VirtualFs for testing. Must be called from a sync context
 /// (not inside #[tokio::test]) because VirtualFs::new() calls block_on internally.
 pub fn make_test_vfs(
@@ -466,12 +476,20 @@ pub fn make_test_vfs(
     opts: TestOpts,
     runtime: &tokio::runtime::Runtime,
 ) -> Arc<crate::virtual_fs::VirtualFs> {
+    let advanced_writes = opts.advanced_writes || opts.overlay;
+
     // Repos need a staging dir for HTTP download cache (open_readonly),
     // even when advanced_writes is disabled (mirrors setup.rs logic).
-    let staging_dir = if opts.advanced_writes || hub.is_repo() {
+    let staging_dir = if advanced_writes || hub.is_repo() {
         let path = std::env::temp_dir().join(format!("hf_mount_test_{}", std::process::id()));
         std::fs::create_dir_all(&path).expect("failed to create temp staging dir");
-        Some(StagingDir::new(&path))
+        if opts.overlay {
+            let overlay_root = std::env::temp_dir().join(format!("hf_mount_overlay_{}", std::process::id()));
+            std::fs::create_dir_all(&overlay_root).expect("failed to create overlay root dir");
+            Some(StagingDir::new_overlay(&path, overlay_root))
+        } else {
+            Some(StagingDir::new(&path))
+        }
     } else {
         None
     };
@@ -483,7 +501,8 @@ pub fn make_test_vfs(
         staging_dir,
         crate::virtual_fs::VfsConfig {
             read_only: opts.read_only,
-            advanced_writes: opts.advanced_writes,
+            advanced_writes,
+            overlay: opts.overlay,
             uid: 1000,
             gid: 1000,
             poll_interval_secs: 0,
@@ -496,3 +515,47 @@ pub fn make_test_vfs(
         },
     )
 }
+
+/// Build a VFS with overlay mode for testing. The given `overlay_root`
+/// is used as the overlay directory, allowing tests to pre-populate
+/// files before VFS creation.
+/// Otherwise a fresh temp dir is created.
+pub fn make_overlay_test_vfs_with_root(
+    hub: Arc<MockHub>,
+    xet: Arc<MockXet>,
+    overlay_root: std::path::PathBuf,
+) -> OverlayTestVfs {
+    let rt = tokio::runtime::Builder::new_multi_thread()
+        .enable_all()
+        .build()
+        .unwrap();
+    let cache_dir = std::env::temp_dir().join(format!("hf_mount_test_{}", std::process::id()));
+    std::fs::create_dir_all(&cache_dir).expect("failed to create temp staging dir");
+    let staging_dir = Some(StagingDir::new_overlay(&cache_dir, overlay_root.clone()));
+
+    let vfs = crate::virtual_fs::VirtualFs::new(
+        rt.handle().clone(),
+        hub,
+        xet,
+        staging_dir,
+        crate::virtual_fs::VfsConfig {
+            read_only: false,
+            advanced_writes: true,
+            overlay: true,
+            uid: 1000,
+            gid: 1000,
+            poll_interval_secs: 0,
+            metadata_ttl: Duration::from_secs(1),
+            serve_lookup_from_cache: false,
+            filter_os_files: true,
+            direct_io: false,
+            flush_debounce: Duration::from_millis(100),
+            flush_max_batch_window: Duration::from_secs(1),
+        },
+    );
+    OverlayTestVfs {
+        runtime: rt,
+        vfs,
+        overlay_root,
+    }
+}

src/virtual_fs/mod.rs

@@ -48,6 +48,7 @@ fn is_os_junk(name: &str) -> bool {
 pub struct VfsConfig {
     pub read_only: bool,
     pub advanced_writes: bool,
+    pub overlay: bool,
     pub uid: u32,
     pub gid: u32,
     pub poll_interval_secs: u64,
@@ -137,7 +138,7 @@ impl VirtualFs {
         let inodes = Arc::new(RwLock::new(InodeTable::new()));
         let negative_cache = Arc::new(RwLock::new(HashMap::new()));
 
-        let flush_manager = if !config.read_only && config.advanced_writes {
+        let flush_manager = if !config.read_only && config.advanced_writes && !config.overlay {
             let sd = staging_dir
                 .as_ref()
                 .expect("--advanced-writes requires a staging directory");
@@ -514,6 +515,62 @@ impl VirtualFs {
             }
         }
 
+        // Overlay mode: merge local directory entries into the inode table.
+        // New local files are inserted; existing remote entries with the same
+        // name are overridden (local size/mtime, marked dirty so reads come
+        // from the local file). Uses the pre-mount fd to access the shadowed dir.
+        if let Some(sd) = &self.staging_dir
+            && let Some(overlay_root) = sd.overlay_root()
+        {
+            let dir_path = inodes.get(parent_ino).map(|e| e.full_path.clone()).unwrap_or_default();
+            let local_dir = overlay_root.join(&dir_path);
+            if let Ok(entries) = std::fs::read_dir(&local_dir) {
+                let now = SystemTime::now();
+                for entry in entries.flatten() {
+                    let name = entry.file_name().to_string_lossy().to_string();
+                    if self.filter_os_files && is_os_junk(&name) {
+                        continue;
+                    }
+                    // Use symlink_metadata to avoid following symlinks
+                    let metadata = match std::fs::symlink_metadata(entry.path()) {
+                        Ok(m) => m,
+                        Err(_) => continue,
+                    };
+                    // Skip symlinks — only regular files and directories
+                    if metadata.is_symlink() {
+                        continue;
+                    }
+                    let kind = if metadata.is_dir() {
+                        InodeKind::Directory
+                    } else {
+                        InodeKind::File
+                    };
+                    let full_path = inode::child_path(&dir_path, &name);
+                    let mtime = metadata.modified().unwrap_or(now);
+                    let size = metadata.len();
+                    let mode = {
+                        use std::os::unix::fs::PermissionsExt;
+                        (metadata.permissions().mode() & 0o777) as u16
+                    };
+                    // insert() returns existing inode if path already exists (e.g. remote file)
+                    let ino = inodes.insert(
+                        parent_ino, name, full_path, kind, size, mtime, None, mode, self.uid, self.gid,
+                    );
+                    // Always update from local file (local overrides remote).
+                    // Clear xet_hash so unlink/rename won't send stale remote ops.
+                    if let Some(e) = inodes.get_mut(ino) {
+                        e.size = size;
+                        e.mtime = mtime;
+                        e.xet_hash = None;
+                        e.set_dirty();
+                        if kind == InodeKind::Directory {
+                            e.children_loaded = false; // will be lazy-loaded on access
+                        }
+                    }
+                }
+            }
+        }
+
         if let Some(parent) = inodes.get_mut(parent_ino) {
             parent.children_loaded = true;
         }
@@ -860,6 +917,17 @@ impl VirtualFs {
             }
         }
         // Advanced-writes: flush staging file to Hub immediately.
+        // In overlay mode, sync the local fd for durability but skip remote upload.
+        if self.staging_dir.as_ref().is_some_and(|sd| sd.is_overlay()) {
+            let files = self.open_files.read().expect("open_files poisoned");
+            if let Some(OpenFile::Local { file, .. }) = files.get(&file_handle) {
+                file.sync_all().map_err(|e| {
+                    error!("Overlay fsync failed for ino={}: {}", ino, e);
+                    libc::EIO
+                })?;
+            }
+            return Ok(());
+        }
         // Note: if the flush_loop is concurrently processing this inode, both may
         // upload the same content. This is benign (idempotent commit, generation-aware
         // clear ensures only one clears dirty).
@@ -931,7 +999,10 @@ impl VirtualFs {
         }
 
         let file_entry = self.get_file_entry(ino)?;
-        let staging_path = self.staging_dir.as_ref().map(|sd| sd.path(ino));
+        let staging_path = self
+            .staging_dir
+            .as_ref()
+            .map(|sd| sd.staging_path(ino, &file_entry.full_path));
 
         if writable && self.advanced_writes {
             // Staging file + async flush (supports random writes and seek)
@@ -959,6 +1030,14 @@ impl VirtualFs {
     ) -> VirtualFsResult<u64> {
         let staging_path = staging_path.expect("staging_dir required for advanced writes");
 
+        // Overlay paths may need parent directories created
+        if let Some(parent) = staging_path.parent() {
+            std::fs::create_dir_all(parent).map_err(|e| {
+                error!("Failed to create staging parent dir {:?}: {}", parent, e);
+                libc::EIO
+            })?;
+        }
+
         // Serialize staging preparation per inode (prevents concurrent download races)
         let staging_mutex = self.staging_lock(ino);
         let _staging_guard = staging_mutex.lock().await;
@@ -1867,7 +1946,15 @@ impl VirtualFs {
                 .staging_dir
                 .as_ref()
                 .expect("staging_dir required for advanced writes")
-                .path(ino);
+                .staging_path(ino, &full_path);
+            // Overlay paths may need parent directories created
+            if let Some(parent) = staging_path.parent()
+                && let Err(e) = std::fs::create_dir_all(parent)
+            {
+                error!("Failed to create staging parent dir {:?}: {}", parent, e);
+                self.inode_table.write().expect("inodes poisoned").remove(ino);
+                return Err(libc::EIO);
+            }
             match OpenOptions::new()
                 .create(true)
                 .truncate(true)
@@ -1981,6 +2068,13 @@ impl VirtualFs {
 
         self.negative_cache_remove(&full_path);
 
+        // Overlay mode: create the directory on disk so it persists
+        if let Some(sd) = &self.staging_dir
+            && let Some(overlay_path) = sd.overlay_root().map(|r| r.join(&full_path))
+        {
+            let _ = std::fs::create_dir_all(&overlay_path);
+        }
+
         let inodes = self.inode_table.read().expect("inodes poisoned");
         Ok(self.make_vfs_attr(inodes.get(ino).ok_or(libc::ENOENT)?))
     }
@@ -2053,7 +2147,7 @@ impl VirtualFs {
 
         // Clean up staging file only if inode was fully removed (no remaining hard links)
         if inode_removed && let Some(ref staging_dir) = self.staging_dir {
-            let staging_path = staging_dir.path(ino);
+            let staging_path = staging_dir.staging_path(ino, &full_path);
             if let Err(e) = std::fs::remove_file(&staging_path)
                 && e.kind() != std::io::ErrorKind::NotFound
             {