Merge branch 'master' into feature/s3fs-path-style-and-bucket-check

Author: huanghantao

.github/workflows/daily-build.yml

@@ -172,6 +172,7 @@ jobs:
           file: ./agfs-server/Dockerfile
           push: true
           tags: |
+            ${{ secrets.DOCKERHUB_USERNAME }}/agfs:latest
             ${{ secrets.DOCKERHUB_USERNAME }}/agfs:nightly
             ${{ secrets.DOCKERHUB_USERNAME }}/agfs:nightly-${{ steps.version.outputs.date }}
             ${{ secrets.DOCKERHUB_USERNAME }}/agfs:nightly-${{ steps.version.outputs.short_sha }}

README.md

@@ -38,7 +38,19 @@ curl -fsSL https://raw.githubusercontent.com/c4pt0r/agfs/master/install.sh | sh
 Or via Docker:
 
 ```bash
-docker pull c4pt0r/agfs-server:latest
+docker pull c4pt0r/agfs:latest
+
+# Run the server (HTTP API only)
+docker run -p 8080:8080 -e SKIP_FUSE_MOUNT=true c4pt0r/agfs:latest
+
+# On Linux, you can enable FUSE mounting with additional privileges
+docker run -p 8080:8080 \
+  --device /dev/fuse \
+  --cap-add SYS_ADMIN \
+  --security-opt apparmor:unconfined \
+  c4pt0r/agfs:latest
+
+# Note: FUSE mounting in Docker is not supported on macOS
 ```
 
 Connect using agfs-shell:
@@ -212,3 +224,4 @@ See [task_loop.py](./agfs-mcp/demos/task_loop.py) for a complete example.
 - [agfs-server](./agfs-server/README.md) - Server configuration and plugin development
 - [agfs-shell](./agfs-shell/README.md) - Interactive shell client
 - [agfs-fuse](./agfs-fuse/README.md) - FUSE filesystem mount (Linux)
+

agfs-fuse/pkg/fusefs/file.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/hanwen/go-fuse/v2/fs"
 	"github.com/hanwen/go-fuse/v2/fuse"
+	log "github.com/sirupsen/logrus"
 )
 
 // AGFSFileHandle represents an open file handle
@@ -32,15 +33,19 @@ func (fh *AGFSFileHandle) Read(ctx context.Context, dest []byte, off int64) (fus
 
 // Write writes data to the file
 func (fh *AGFSFileHandle) Write(ctx context.Context, data []byte, off int64) (written uint32, errno syscall.Errno) {
+	path := fh.node.getPath()
+	log.Debugf("[file] Write called: path=%s, len=%d, off=%d, handle=%d", path, len(data), off, fh.handle)
+
 	n, err := fh.node.root.handles.Write(fh.handle, data, off)
 	if err != nil {
+		log.Errorf("[file] Write failed: path=%s, err=%v", path, err)
 		return 0, syscall.EIO
 	}
 
 	// Invalidate metadata cache since file size may have changed
-	path := fh.node.getPath()
 	fh.node.root.metaCache.Invalidate(path)
 
+	log.Debugf("[file] Write success: path=%s, written=%d", path, n)
 	return uint32(n), 0
 }
 

agfs-fuse/pkg/fusefs/handles.go

@@ -408,12 +408,16 @@ func (hm *HandleManager) Write(fuseHandle uint64, data []byte, offset int64) (in
 	path := info.path
 	hm.mu.Unlock()
 
+	log.Debugf("[handles] Local handle write: path=%s, len=%d, offset=%d", path, len(data), offset)
+
 	// Send directly to server
 	_, err := hm.client.Write(path, data)
 	if err != nil {
+		log.Errorf("[handles] Write failed for %s: %v", path, err)
 		return 0, fmt.Errorf("failed to write to server: %w", err)
 	}
 
+	log.Debugf("[handles] Write success for %s: %d bytes", path, len(data))
 	return len(data), nil
 }
 

agfs-fuse/pkg/fusefs/node.go

@@ -8,6 +8,7 @@ import (
 	agfs "github.com/c4pt0r/agfs/agfs-sdk/go"
 	"github.com/hanwen/go-fuse/v2/fs"
 	"github.com/hanwen/go-fuse/v2/fuse"
+	log "github.com/sirupsen/logrus"
 )
 
 // AGFSNode represents a file or directory node
@@ -250,25 +251,34 @@ func (n *AGFSNode) Create(ctx context.Context, name string, flags uint32, mode u
 	path := n.getPath()
 	childPath := filepath.Join(path, name)
 
+	log.Debugf("[node] Create called: path=%s, name=%s, childPath=%s", path, name, childPath)
+
 	// Create the file
 	err := n.root.client.Create(childPath)
 	if err != nil {
+		log.Errorf("[node] Create failed for %s: %v", childPath, err)
 		return nil, nil, 0, syscall.EIO
 	}
 
+	log.Debugf("[node] Create succeeded, opening handle for %s", childPath)
+
 	// Invalidate caches
 	n.root.invalidateCache(childPath)
 
 	// Open the file with the requested flags
 	openFlags := convertOpenFlags(flags)
 	fuseHandle, err := n.root.handles.Open(childPath, openFlags, mode)
 	if err != nil {
+		log.Errorf("[node] Open handle failed for %s: %v", childPath, err)
 		return nil, nil, 0, syscall.EIO
 	}
 
+	log.Debugf("[node] Handle opened: %d for %s", fuseHandle, childPath)
+
 	// Fetch file info
 	info, err := n.root.client.Stat(childPath)
 	if err != nil {
+		log.Errorf("[node] Stat failed for %s: %v", childPath, err)
 		n.root.handles.Close(fuseHandle)
 		return nil, nil, 0, syscall.EIO
 	}

agfs-sdk/python/pyagfs/client.py

@@ -566,7 +566,7 @@ def get_plugins_info(self) -> List[dict]:
         except Exception as e:
             self._handle_request_error(e)
 
-    def grep(self, path: str, pattern: str, recursive: bool = False, case_insensitive: bool = False, stream: bool = False):
+    def grep(self, path: str, pattern: str, recursive: bool = False, case_insensitive: bool = False, stream: bool = False, limit: int = 0):
         """Search for a pattern in files using regular expressions
 
         Args:
@@ -575,6 +575,7 @@ def grep(self, path: str, pattern: str, recursive: bool = False, case_insensitiv
             recursive: Whether to search recursively in directories (default: False)
             case_insensitive: Whether to perform case-insensitive matching (default: False)
             stream: Whether to stream results as NDJSON (default: False)
+            limit: Maximum number of results to return (0 means default, for vector search defaults to 10)
 
         Returns:
             If stream=False: Dict with 'matches' (list of match objects) and 'count'
@@ -593,15 +594,19 @@ def grep(self, path: str, pattern: str, recursive: bool = False, case_insensitiv
             ...         print(f"{item['file']}:{item['line']}: {item['content']}")
         """
         try:
+            request_body = {
+                "path": path,
+                "pattern": pattern,
+                "recursive": recursive,
+                "case_insensitive": case_insensitive,
+                "stream": stream
+            }
+            if limit > 0:
+                request_body["limit"] = limit
+
             response = self.session.post(
                 f"{self.api_base}/grep",
-                json={
-                    "path": path,
-                    "pattern": pattern,
-                    "recursive": recursive,
-                    "case_insensitive": case_insensitive,
-                    "stream": stream
-                },
+                json=request_body,
                 timeout=None if stream else self.timeout,
                 stream=stream
             )

agfs-server/Dockerfile

@@ -3,7 +3,7 @@ FROM golang:1.25-alpine AS builder
 
 # Install build dependencies
 RUN apk add --no-cache git make gcc musl-dev sqlite-dev python3 py3-pip npm \
-    autoconf automake libtool python3-dev jq-dev
+    autoconf automake libtool python3-dev jq-dev fuse3-dev
 
 # Install uv for Python package management
 RUN pip3 install --break-system-packages uv
@@ -29,11 +29,16 @@ WORKDIR /build-shell
 COPY agfs-shell .
 RUN python3 build.py
 
+# Build agfs-fuse
+WORKDIR /build-fuse
+COPY agfs-fuse .
+RUN go build -o agfs-fuse ./cmd/agfs-fuse
+
 # Runtime stage
 FROM alpine:latest
 
-# Install runtime dependencies (including Python 3 for agfs-shell)
-RUN apk add --no-cache ca-certificates sqlite-libs python3 jq oniguruma
+# Install runtime dependencies (including Python 3 for agfs-shell and fuse3 for agfs-fuse)
+RUN apk add --no-cache ca-certificates sqlite-libs python3 jq oniguruma fuse3 wget
 
 # Create app directory
 WORKDIR /app
@@ -48,13 +53,20 @@ COPY --from=builder /build/config.example.yaml /config.yaml
 # Copy agfs-shell portable distribution
 COPY --from=builder /build-shell/dist/agfs-shell-portable /usr/local/agfs-shell
 RUN ln -s /usr/local/agfs-shell/agfs-shell /usr/local/bin/agfs-shell
+RUN ln -s /usr/local/agfs-shell/agfs-shell /usr/local/bin/agfs
+
+# Copy agfs-fuse binary
+COPY --from=builder /build-fuse/agfs-fuse /usr/local/bin/agfs-fuse
+
+# Copy docker entrypoint script
+COPY agfs-server/docker-entrypoint.sh /docker-entrypoint.sh
+RUN chmod +x /docker-entrypoint.sh
 
-# Create directory for localfs mount
-RUN mkdir -p /data
+# Create directories
+RUN mkdir -p /data /mnt/agfs
 
 # Expose default port
 EXPOSE 8080
 
-# Run the server
-ENTRYPOINT ["./agfs-server"]
-CMD ["-c", "/config.yaml"]
+# Run agfs-server with agfs-fuse mount
+ENTRYPOINT ["/docker-entrypoint.sh"]

agfs-server/README.md

@@ -20,22 +20,22 @@ The easiest way to get started is using Docker:
 
 1.  **Pull the image**:
     ```bash
-    docker pull c4pt0r/agfs-server:latest
+    docker pull c4pt0r/agfs:latest
     ```
 
 2.  **Run the server with port mapping**:
     ```bash
     # Basic run - expose port 8080 to host
-    docker run -d -p 8080:8080 --name agfs-server c4pt0r/agfs-server:latest
+    docker run -d -p 8080:8080 --name agfs-server c4pt0r/agfs:latest
 
     # With custom port mapping (host:container)
-    docker run -d -p 9000:8080 --name agfs-server c4pt0r/agfs-server:latest
+    docker run -d -p 9000:8080 --name agfs-server c4pt0r/agfs:latest
 
     # With data persistence (mount /data directory)
-    docker run -d -p 8080:8080 -v $(pwd)/data:/data --name agfs-server c4pt0r/agfs-server:latest
+    docker run -d -p 8080:8080 -v $(pwd)/data:/data --name agfs-server c4pt0r/agfs:latest
 
     # With custom configuration
-    docker run -d -p 8080:8080 -v $(pwd)/config.yaml:/config.yaml --name agfs-server c4pt0r/agfs-server:latest
+    docker run -d -p 8080:8080 -v $(pwd)/config.yaml:/config.yaml --name agfs-server c4pt0r/agfs:latest
     ```
 
 3.  **Using agfs-shell inside the container**:

agfs-server/cmd/server/main.go

@@ -13,6 +13,7 @@ import (
 	"github.com/c4pt0r/agfs/agfs-server/pkg/mountablefs"
 	"github.com/c4pt0r/agfs/agfs-server/pkg/plugin"
 	"github.com/c4pt0r/agfs/agfs-server/pkg/plugin/api"
+	"github.com/c4pt0r/agfs/agfs-server/pkg/plugins/devfs"
 	"github.com/c4pt0r/agfs/agfs-server/pkg/plugins/gptfs"
 	"github.com/c4pt0r/agfs/agfs-server/pkg/plugins/heartbeatfs"
 	"github.com/c4pt0r/agfs/agfs-server/pkg/plugins/hellofs"
@@ -28,6 +29,7 @@ import (
 	"github.com/c4pt0r/agfs/agfs-server/pkg/plugins/sqlfs2"
 	"github.com/c4pt0r/agfs/agfs-server/pkg/plugins/streamfs"
 	"github.com/c4pt0r/agfs/agfs-server/pkg/plugins/streamrotatefs"
+	"github.com/c4pt0r/agfs/agfs-server/pkg/plugins/vectorfs"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -43,6 +45,7 @@ type PluginFactory func() plugin.ServicePlugin
 
 // availablePlugins maps plugin names to their factory functions
 var availablePlugins = map[string]PluginFactory{
+	"devfs":          func() plugin.ServicePlugin { return devfs.NewDevFSPlugin() },
 	"serverinfofs":   func() plugin.ServicePlugin { return serverinfofs.NewServerInfoFSPlugin() },
 	"memfs":          func() plugin.ServicePlugin { return memfs.NewMemFSPlugin() },
 	"queuefs":        func() plugin.ServicePlugin { return queuefs.NewQueueFSPlugin() },
@@ -58,6 +61,7 @@ var availablePlugins = map[string]PluginFactory{
 	"sqlfs2":         func() plugin.ServicePlugin { return sqlfs2.NewSQLFS2Plugin() },
 	"localfs":        func() plugin.ServicePlugin { return localfs.NewLocalFSPlugin() },
 	"gptfs":          func() plugin.ServicePlugin { return gptfs.NewGptfs() },
+	"vectorfs":       func() plugin.ServicePlugin { return vectorfs.NewVectorFSPlugin() },
 }
 
 const sampleConfig = `# AGFS Server Configuration File
@@ -329,6 +333,22 @@ func main() {
 		}
 	}
 
+	// Mount DevFS by default (always enabled)
+	log.Info("Mounting default DevFS at /dev...")
+	devfsPlugin := devfs.NewDevFSPlugin()
+	devfsConfig := map[string]interface{}{
+		"mount_path": "/dev",
+	}
+	if err := devfsPlugin.Validate(devfsConfig); err != nil {
+		log.Errorf("Failed to validate DevFS: %v", err)
+	} else if err := devfsPlugin.Initialize(devfsConfig); err != nil {
+		log.Errorf("Failed to initialize DevFS: %v", err)
+	} else if err := mfs.Mount("/dev", devfsPlugin); err != nil {
+		log.Errorf("Failed to mount DevFS at /dev: %v", err)
+	} else {
+		log.Info("DevFS mounted successfully at /dev")
+	}
+
 	// Mount all enabled plugins
 	log.Info("Mounting plugin filesytems...")
 	for pluginName, pluginCfg := range cfg.Plugins {

agfs-server/config.example.yaml

@@ -150,6 +150,41 @@ plugins:
 #        secret_access_key: secret
 #        prefix: agfs/ # Optional: all keys will be prefixed with "agfs/"
 #
+#  # ============================================================================
+#  # VectorFS - Document Vector Search (AI-Powered Semantic Search)
+#  # ============================================================================
+#  # VectorFS provides semantic search capabilities for documents using:
+#  # - S3 for scalable document storage
+#  # - TiDB Cloud vector index for fast similarity search
+#  # - OpenAI embeddings for vector representations
+#  #
+#  # See config.vectorfs.example.yaml for detailed configuration
+#  #
+#  vectorfs:
+#    enabled: false
+#    path: /vectorfs
+#    config:
+#      # S3 Configuration
+#      s3_bucket: "your-bucket-name"
+#      s3_key_prefix: "vectorfs"
+#      s3_region: "us-west-1"
+#      s3_access_key: "your-access-key"
+#      s3_secret_key: "your-secret-key"
+#
+#      # TiDB Cloud Configuration
+#      tidb_dsn: "user:pass@tcp(host:4000)/db?tls=true"
+#
+#      # OpenAI Configuration
+#      openai_api_key: "sk-..."
+#      embedding_model: "text-embedding-3-small"
+#      embedding_dim: 1536
+#
+#      # Performance tuning (optional)
+#      chunk_size: 512
+#      chunk_overlap: 50
+#      index_workers: 4
+#
+
 #  # ============================================================================
 #  # HTTPFS - HTTP File Server (Multiple Instances)
 #  # ============================================================================
@@ -231,6 +266,10 @@ plugins:
 # ├── sqlfs_tidb/     (database-backed filesystem - TiDB)
 # ├── httagfs-queue    (virtual status file - httagfs serving queuefs)
 # ├── httagfs-s3       (virtual status file - httagfs serving S3)
+# ├── vectorfs/       (AI-powered semantic document search)
+# │   ├── README      (VectorFS documentation)
+# │   └── <namespace>/
+# │       └── docs/   (indexed documents with semantic search)
 # ├── s3fs/
 # │   └── aws/        (AWS S3 storage)
 # └── proxyfs/
@@ -287,3 +326,25 @@ plugins:
 #   agfs mount httagfs /docs agfs_path=/memfs/docs http_port=8001
 #   agfs mount httagfs /images agfs_path=/memfs/images http_port=8002
 #   agfs mount httagfs /s3-public agfs_path=/s3fs/aws/public http_port=8003
+#
+# VectorFS - AI-Powered Semantic Search (when enabled):
+#   # Create a namespace
+#   agfs mkdir /vectorfs/my_docs
+#
+#   # Add documents (auto-indexed with embeddings)
+#   agfs write /vectorfs/my_docs/docs/guide.txt "Kubernetes deployment guide..."
+#   agfs write /vectorfs/my_docs/docs/tutorials/docker.txt "Docker tutorial..."
+#
+#   # Copy entire folders (async indexing, no timeout)
+#   agfs cp -r /s3fs/aws/docs /vectorfs/my_docs/docs/imported
+#
+#   # Read original documents
+#   agfs cat /vectorfs/my_docs/docs/guide.txt
+#   agfs cat /vectorfs/my_docs/docs/tutorials/docker.txt
+#
+#   # Semantic search (vector similarity, returns relevant chunks)
+#   agfs grep "how to deploy containers" /vectorfs/my_docs/docs
+#   # Returns results with similarity scores
+#
+#   # Use agfs-shell's fsgrep for better formatted output
+#   fsgrep -r "deployment strategies" /vectorfs/my_docs/docs