Merge pull request #10 from huanghantao/feature/s3fs-path-style-and-bucket-check

Feature/s3fs path style and bucket check

Author: c4pt0r

agfs-server/pkg/plugins/s3fs/client.go

@@ -41,6 +41,18 @@ type S3Config struct {
 	Endpoint        string // Optional custom endpoint (for S3-compatible services)
 	Prefix          string // Optional prefix for all keys (will be wrapped for isolation)
 	DisableSSL      bool   // For testing with local S3
+	UsePathStyle    bool   // Use path-style requests (required for MinIO and some S3-compatible services)
+}
+
+// checkBucketAccess verifies that the bucket exists and is accessible
+// Uses ListObjectsV2 instead of HeadBucket for better compatibility
+// Some S3-compatible services (e.g., certain object storage providers) don't support HeadBucket
+func checkBucketAccess(ctx context.Context, client *s3.Client, bucket string) error {
+	_, err := client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
+		Bucket:  aws.String(bucket),
+		MaxKeys: aws.Int32(1), // Only need to check if we can access the bucket
+	})
+	return err
 }
 
 // NewS3Client creates a new S3 client
@@ -74,17 +86,21 @@ func NewS3Client(cfg S3Config) (*S3Client, error) {
 	if cfg.Endpoint != "" {
 		clientOpts = append(clientOpts, func(o *s3.Options) {
 			o.BaseEndpoint = aws.String(cfg.Endpoint)
-			o.UsePathStyle = true // Required for MinIO and some S3-compatible services
+		})
+	}
+
+	// Set path-style requests if configured (required for MinIO and some S3-compatible services)
+	if cfg.UsePathStyle {
+		clientOpts = append(clientOpts, func(o *s3.Options) {
+			o.UsePathStyle = true
 		})
 	}
 
 	client := s3.NewFromConfig(awsCfg, clientOpts...)
 
-	// Verify bucket exists
-	_, err = client.HeadBucket(ctx, &s3.HeadBucketInput{
-		Bucket: aws.String(cfg.Bucket),
-	})
-	if err != nil {
+	// Verify bucket exists using ListObjectsV2 (more compatible than HeadBucket)
+	// Some object storage services don't support HeadBucket
+	if err = checkBucketAccess(ctx, client, cfg.Bucket); err != nil {
 		return nil, fmt.Errorf("failed to access bucket %s: %w", cfg.Bucket, err)
 	}
 

agfs-server/pkg/plugins/s3fs/s3fs.go

@@ -519,7 +519,7 @@ func (p *S3FSPlugin) Validate(cfg map[string]interface{}) error {
 	// Check for unknown parameters
 	allowedKeys := []string{
 		"bucket", "region", "access_key_id", "secret_access_key", "endpoint", "prefix", "disable_ssl", "mount_path",
-		"cache_enabled", "cache_ttl", "stat_cache_ttl", "cache_max_size",
+		"cache_enabled", "cache_ttl", "stat_cache_ttl", "cache_max_size", "use_path_request_style",
 	}
 	if err := config.ValidateOnlyKnownKeys(cfg, allowedKeys); err != nil {
 		return err
@@ -538,7 +538,7 @@ func (p *S3FSPlugin) Validate(cfg map[string]interface{}) error {
 	}
 
 	// Validate boolean parameters
-	for _, key := range []string{"disable_ssl", "cache_enabled"} {
+	for _, key := range []string{"disable_ssl", "use_path_request_style", "cache_enabled"} {
 		if err := config.ValidateBoolType(cfg, key); err != nil {
 			return err
 		}
@@ -559,6 +559,7 @@ func (p *S3FSPlugin) Initialize(config map[string]interface{}) error {
 		Endpoint:        getStringConfig(config, "endpoint", ""),
 		Prefix:          getStringConfig(config, "prefix", ""),
 		DisableSSL:      getBoolConfig(config, "disable_ssl", false),
+		UsePathStyle:    getBoolConfig(config, "use_path_request_style", false),
 	}
 
 	if cfg.Bucket == "" {
@@ -643,6 +644,13 @@ func (p *S3FSPlugin) GetConfigParams() []plugin.ConfigParameter {
 			Default:     "false",
 			Description: "Disable SSL for S3 connections",
 		},
+		{
+			Name:        "use_path_request_style",
+			Type:        "bool",
+			Required:    false,
+			Default:     "false",
+			Description: "Use path-style requests instead of virtual-hosted-style (required for MinIO and some S3-compatible services)",
+		},
 		{
 			Name:        "cache_enabled",
 			Type:        "bool",
@@ -718,6 +726,7 @@ CONFIGURATION:
     secret_access_key = "minioadmin"
     endpoint = "http://localhost:9000"
     disable_ssl = true
+    use_path_request_style = true  # Required for MinIO and some S3-compatible services
 
   Multiple S3 Buckets:
   [plugins.s3fs_prod]