Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
104 changes: 90 additions & 14 deletions crates/ragfs/src/plugins/s3fs/client.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ use aws_sdk_s3::config::http::HttpResponse;
use aws_sdk_s3::config::{BehaviorVersion, Credentials, Region};
use aws_sdk_s3::error::ProvideErrorMetadata;
use aws_sdk_s3::error::SdkError;
use aws_sdk_s3::operation::put_object::builders::PutObjectFluentBuilder;
use aws_sdk_s3::operation::put_object::{PutObjectError, PutObjectOutput};
use aws_sdk_s3::operation::{RequestId, RequestIdExt};
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::Client;
Expand All @@ -18,6 +20,34 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH};
const ENCODED_SEGMENT_PREFIX: char = '!';
const HEX_UPPER: &[u8; 16] = b"0123456789ABCDEF";

/// S3-compatible vendor behavior selected by configuration.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(super) enum S3Vendor {
/// Standard S3-compatible behavior.
Standard,
/// Alibaba Cloud OSS S3-compatible behavior.
AliyunOss,
}

impl S3Vendor {
/// Parse the S3 vendor option from plugin configuration.
///
/// # Arguments
/// * `config` - S3FS plugin configuration.
///
/// # Returns
/// The configured S3 vendor, or `Standard` when omitted.
pub(super) fn from_config(config: &HashMap<String, ConfigValue>) -> Result<Self> {
match config.get("s3_vendor").and_then(|v| v.as_string()) {
None | Some("") | Some("standard") => Ok(Self::Standard),
Some("aliyun_oss") => Ok(Self::AliyunOss),
Some(v) => Err(Error::config(format!(
"invalid s3_vendor: {v}; expected one of: standard, aliyun_oss"
))),
}
}
}

fn partial_delete_error(bucket: &str, errors: &[aws_sdk_s3::types::Error]) -> Option<Error> {
if errors.is_empty() {
return None;
Expand Down Expand Up @@ -369,6 +399,7 @@ pub struct S3Client {
marker_mode: DirectoryMarkerMode,
disable_batch_delete: bool,
auto_detect_content_type: bool,
s3_vendor: S3Vendor,
}

impl S3Client {
Expand Down Expand Up @@ -397,6 +428,8 @@ impl S3Client {
.unwrap_or("us-east-1")
.to_string();

let s3_vendor = S3Vendor::from_config(config)?;

let raw_endpoint = config.get("endpoint").and_then(|v| v.as_string());
let use_ssl = if let Some(v) = config.get("use_ssl").and_then(|v| v.as_bool()) {
v
Expand Down Expand Up @@ -486,6 +519,7 @@ impl S3Client {
marker_mode,
disable_batch_delete,
auto_detect_content_type,
s3_vendor,
})
}

Expand Down Expand Up @@ -655,7 +689,6 @@ impl S3Client {
.put_object()
.bucket(&self.bucket)
.key(key)
.if_none_match("*")
.body(ByteStream::from(data));

if self.auto_detect_content_type {
Expand All @@ -664,17 +697,19 @@ impl S3Client {
}
}

request.send().await.map_err(|e| {
if is_s3_conditional_failure(&e) {
Error::already_exists(key)
} else {
format_sdk_s3_error(
"PutObjectCreateNew",
&format!("bucket={} key={key}", self.bucket),
&e,
)
}
})?;
self.if_none_match_adapter(request, "*")
.await
.map_err(|e| {
if is_s3_conditional_failure(&e) {
Error::already_exists(key)
} else {
format_sdk_s3_error(
"PutObjectCreateNew",
&format!("bucket={} key={key}", self.bucket),
&e,
)
}
})?;

Ok(())
}
Expand All @@ -686,7 +721,6 @@ impl S3Client {
.put_object()
.bucket(&self.bucket)
.key(key)
.if_match(etag)
.body(ByteStream::from(data));

if self.auto_detect_content_type {
Expand All @@ -695,7 +729,7 @@ impl S3Client {
}
}

request.send().await.map_err(|e| {
self.if_match_adapter(request, etag, key).await.map_err(|e| {
if is_s3_conditional_failure(&e) {
Error::AlreadyExists(key.to_string())
} else {
Expand All @@ -714,6 +748,46 @@ impl S3Client {
})
}

/// Apply create-if-absent behavior to a PutObject request.
async fn if_none_match_adapter(
&self,
request: PutObjectFluentBuilder,
value: &str,
) -> std::result::Result<PutObjectOutput, SdkError<PutObjectError, HttpResponse>> {
match self.s3_vendor {
S3Vendor::Standard => request.if_none_match(value).send().await,
S3Vendor::AliyunOss => {
request
.customize()
.mutate_request(|req| {
req.headers_mut().insert("x-oss-forbid-overwrite", "true");
})
.send()
.await
}
}
}

/// Apply match-current-ETag behavior to a PutObject request when supported.
async fn if_match_adapter(
&self,
request: PutObjectFluentBuilder,
etag: &str,
key: &str,
) -> std::result::Result<PutObjectOutput, SdkError<PutObjectError, HttpResponse>> {
match self.s3_vendor {
S3Vendor::Standard => request.if_match(etag).send().await,
S3Vendor::AliyunOss => {
tracing::info!(
bucket = %self.bucket,
key = %key,
"aliyun_oss does not support PutObject If-Match; proceeding without CAS guarantee"
);
request.send().await
}
}
}

/// Delete a single object
pub async fn delete_object(&self, key: &str) -> Result<()> {
self.client
Expand Down Expand Up @@ -1227,6 +1301,7 @@ mod tests {
}
}

/// Build a test S3 client without performing network setup.
fn test_client(prefix: &str, normalize_encoding_chars: &str) -> S3Client {
S3Client {
client: Client::from_conf(
Expand All @@ -1241,6 +1316,7 @@ mod tests {
marker_mode: DirectoryMarkerMode::Empty,
disable_batch_delete: false,
auto_detect_content_type: false,
s3_vendor: S3Vendor::Standard,
}
}

Expand Down
14 changes: 12 additions & 2 deletions crates/ragfs/src/plugins/s3fs/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ use std::sync::{Arc, Mutex};
use std::time::SystemTime;

use cache::{S3ListDirCache, S3StatCache};
use client::{ListTreePage, S3Client};
use client::{ListTreePage, S3Client, S3Vendor};
use futures::stream::{self, StreamExt};
use regex::Regex;
use std::sync::atomic::{AtomicUsize, Ordering};
Expand Down Expand Up @@ -1175,6 +1175,12 @@ impl S3FSPlugin {
"true",
"Use path-style addressing (bucket/key vs bucket.host/key)",
),
ConfigParameter::optional(
"s3_vendor",
"string",
"standard",
"S3 vendor behavior: standard, aliyun_oss",
),
ConfigParameter::optional(
"prefix",
"string",
Expand Down Expand Up @@ -1320,7 +1326,9 @@ plugins:
config:
bucket: my-oss-bucket
region: cn-beijing
endpoint: http://s3.oss-cn-beijing.aliyuncs.com
endpoint: https://s3.oss-cn-beijing.aliyuncs.com
s3_vendor: aliyun_oss
use_path_style: false
disable_batch_delete: true
```

Expand Down Expand Up @@ -1377,6 +1385,8 @@ plugins:
}
}

S3Vendor::from_config(&config.params)?;

if let Some(value) = config.params.get("auto_detect_content_type") {
if value.as_bool().is_none() {
return Err(Error::config(
Expand Down
13 changes: 13 additions & 0 deletions docs/en/guides/01-configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -1320,6 +1320,7 @@ Code entry: `openviking/session/auto_commit_policy.py:AutoCommitPolicy`.
| `prefix` | str | Optional key prefix for namespace isolation | "" |
| `use_ssl` | bool | Enable/disable SSL (HTTPS) for S3 connections. Also controls the scheme auto-prefixed onto bare-hostname `endpoint` values | true |
| `use_path_style` | bool | true for PathStyle used by MinIO and some S3-compatible services; false for VirtualHostStyle used by TOS and some S3-compatible services | true |
| `s3_vendor` | str | S3 vendor behavior: `standard` or `aliyun_oss` | `"standard"` |
| `auto_detect_content_type` | bool | Automatically infer MIME type from the object key / filename extension and set the S3 object `Content-Type` header during upload | false |
| `directory_marker_mode` | str | How to persist directory markers: `none`, `empty`, or `nonempty` | `"empty"` |
| `normalize_encoding_chars` | str | Characters to escape in S3 object keys as `!HH` hexadecimal bytes; empty string disables normalization | `"?#%+@"` |
Expand All @@ -1336,6 +1337,18 @@ Typical choices:
- For TOS or other VirtualHostStyle backends that reject zero-byte directory markers, use `nonempty`.
- If you want pure prefix-style behavior and do not need persisted empty directories, use `none`.

`s3_vendor` controls vendor-specific request behavior:

- `standard` is the default. It uses standard S3 conditional headers.
- `aliyun_oss` is for Alibaba Cloud OSS.
- For create-if-absent writes, OSS uses `x-oss-forbid-overwrite: true`.
- For overwrite writes, OSS does not send `If-Match`.
- The write still proceeds and emits an `INFO` log.
- CAS is not guaranteed in this case. CAS means write-after-match.
- `s3_vendor` does not change other options.
- Set `use_path_style` explicitly.
- Set `disable_batch_delete` explicitly.

`normalize_encoding_chars` controls which characters RAGFS rewrites before issuing S3 requests:

- The default value is `"?#%+@"`, so only `?`, `#`, `%`, `+`, and `@` are escaped.
Expand Down
27 changes: 27 additions & 0 deletions docs/en/guides/13-multi-write-storage.md
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,7 @@ When using S3-compatible services (MinIO, RustFS, Ceph, etc.), the `s3` section
| Field | Required? | Description |
| --- | --- | --- |
| `use_path_style` | Yes for most S3-compatible | Set to `true` for path-style URLs (`http://host/bucket/key`). Most S3-compatible services require this. |
| `s3_vendor` | Required for Alibaba Cloud OSS | Set to `aliyun_oss`. Create-if-absent writes use `x-oss-forbid-overwrite: true`. |
| `directory_marker_mode` | Yes for S3-compatible | **Must be explicitly set to `"none"`**. Without this, the RAGFS Rust binding panics with `AGFSConfigError: invalid directory_marker_mode: null` during startup, causing a silent crash loop. |
| `use_ssl` | Optional | Set to `false` for HTTP endpoints (e.g. `http://localhost:9000`). |

Expand All @@ -121,6 +122,32 @@ When using S3-compatible services (MinIO, RustFS, Ceph, etc.), the `s3` section
}
```

**Alibaba Cloud OSS example:**

```json
{
"name": "oss-backup",
"backend": "s3",
"s3": {
"bucket": "my-oss-bucket",
"endpoint": "https://s3.oss-cn-beijing.aliyuncs.com",
"region": "cn-beijing",
"access_key": "your-access-key",
"secret_key": "your-secret-key",
"prefix": "openviking",
"use_path_style": false,
"s3_vendor": "aliyun_oss",
"disable_batch_delete": true
}
}
```

> `aliyun_oss` does not change other options.
> Set `use_path_style` and `disable_batch_delete` explicitly.
> OSS does not support `PutObject If-Match`.
> OpenViking continues the write and emits an `INFO` log.
> CAS is not guaranteed on that path.

> **Why is `directory_marker_mode` required?**
>
> S3-compatible storage services handle "directories" differently from AWS S3. The RAGFS Rust binding must know whether to write directory marker objects when creating directories. Valid values are `"none"`, `"empty"`, and `"nonempty"`. For S3-compatible services that don't use directory markers (RustFS, MinIO, Ceph, etc.), set this to `"none"`. If omitted, the Rust binding defaults to `null` which is rejected, causing the server to crash silently during startup with `AGFSConfigError: invalid directory_marker_mode: null`.
Expand Down
13 changes: 13 additions & 0 deletions docs/zh/guides/01-configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -1295,6 +1295,7 @@ RAGFS 默认使用 Rust binding 模式,通过 Rust 实现直接访问文件系
| `prefix` | str | 用于命名空间隔离的可选键前缀 | "" |
| `use_ssl` | bool | 为 S3 连接启用/禁用 SSL(HTTPS)。也用于决定 `endpoint` 仅填主机名时自动补的协议前缀 | true |
| `use_path_style` | bool | true 表示对 MinIO 和某些 S3 兼容服务使用 PathStyle;false 表示对 TOS 和某些 S3 兼容服务使用 VirtualHostStyle | true |
| `s3_vendor` | str | S3 厂商行为,可选 `standard`、`aliyun_oss` | `"standard"` |
| `auto_detect_content_type` | bool | 上传时根据 object key / 文件名后缀自动推断 MIME 类型,并写入 S3 对象的 `Content-Type` | false |
| `directory_marker_mode` | str | 目录 marker 的持久化方式,可选 `none`、`empty`、`nonempty` | `"empty"` |
| `normalize_encoding_chars` | str | 需要在 S3 object key 中转义为 `!HH` 十六进制字节的字符集合;空字符串表示关闭编码 | `"?#%+@"` |
Expand All @@ -1311,6 +1312,18 @@ RAGFS 默认使用 Rust binding 模式,通过 Rust 实现直接访问文件系
- 对 TOS 或其他拒绝 0 字节目录 marker 的 VirtualHostStyle 后端,使用 `nonempty`。
- 如果你想完全使用 prefix 风格行为,并且不需要持久化空目录,可以使用 `none`。

`s3_vendor` 控制厂商请求差异:

- `standard` 是默认值。使用标准 S3 条件头。
- `aliyun_oss` 用于 Alibaba Cloud OSS。
- OSS 创建对象时,使用 `x-oss-forbid-overwrite: true`。
- OSS 覆盖写时,不发送 `If-Match`。
- 覆盖写会继续执行,并打印 `INFO` 日志。
- 此时无法保证 CAS。CAS 指按旧值匹配后再写。
- `s3_vendor` 不会修改其他配置。
- `use_path_style` 仍需显式设置。
- `disable_batch_delete` 仍需显式设置。

`normalize_encoding_chars` 用来控制 RAGFS 在发起 S3 请求前需要重写哪些字符:

- 默认值是 `"?#%+@"`,所以只会转义 `?`、`#`、`%`、`+`、`@`。
Expand Down
27 changes: 27 additions & 0 deletions docs/zh/guides/13-multi-write-storage.md
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,7 @@
| 字段 | 是否必填 | 说明 |
| --- | --- | --- |
| `use_path_style` | 大多数 S3 兼容服务必填 | 设置为 `true` 使用路径风格 URL(`http://host/bucket/key`)。大多数 S3 兼容服务需要此配置。 |
| `s3_vendor` | Alibaba Cloud OSS 必填 | 设置为 `aliyun_oss`。创建对象会使用 `x-oss-forbid-overwrite: true`。 |
| `directory_marker_mode` | S3 兼容服务必填 | **必须显式设置为 `"none"`**。如果不配置,RAGFS Rust binding 启动时会报 `AGFSConfigError: invalid directory_marker_mode: null` 并静默崩溃。 |
| `use_ssl` | 可选 | HTTP 端点(如 `http://localhost:9000`)需要设置为 `false`。 |

Expand All @@ -121,6 +122,32 @@
}
```

**Alibaba Cloud OSS 示例:**

```json
{
"name": "oss-backup",
"backend": "s3",
"s3": {
"bucket": "my-oss-bucket",
"endpoint": "https://s3.oss-cn-beijing.aliyuncs.com",
"region": "cn-beijing",
"access_key": "your-access-key",
"secret_key": "your-secret-key",
"prefix": "openviking",
"use_path_style": false,
"s3_vendor": "aliyun_oss",
"disable_batch_delete": true
}
}
```

> `aliyun_oss` 不会联动其他配置。
> `use_path_style` 和 `disable_batch_delete` 仍需显式设置。
> OSS 不支持 `PutObject If-Match`。
> OpenViking 会继续写入,并打印 `INFO` 日志。
> 该路径无法保证 CAS。

> **为什么需要 `directory_marker_mode`?**
>
> S3 兼容存储服务对"目录"的处理方式与 AWS S3 不同。RAGFS Rust binding 必须知道创建目录时是否需要写入目录标记对象。合法取值为 `"none"`、`"empty"` 和 `"nonempty"`。对于不使用目录标记的 S3 兼容服务(RustFS、MinIO、Ceph 等),设置为 `"none"`。如果省略,Rust binding 默认值为 `null`(不合法),导致服务端在启动时静默崩溃,报错 `AGFSConfigError: invalid directory_marker_mode: null`。
Expand Down
1 change: 1 addition & 0 deletions openviking/utils/agfs_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -383,6 +383,7 @@ def _serialize_s3_plugin_params(s3_config: Any) -> Dict[str, Any]:
"prefix": _get_config_value(s3_config, "prefix", ""),
"disable_ssl": not _get_config_value(s3_config, "use_ssl", True),
"use_path_style": _get_config_value(s3_config, "use_path_style", True),
"s3_vendor": _get_config_value(s3_config, "s3_vendor", "standard"),
"directory_marker_mode": directory_marker_mode.value
if hasattr(directory_marker_mode, "value")
else directory_marker_mode,
Expand Down
5 changes: 5 additions & 0 deletions openviking_cli/utils/config/agfs_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,11 @@ class S3Config(BaseModel):
description="true represent UsePathStyle for MinIO and some S3-compatible services; false represent VirtualHostStyle for TOS and some S3-compatible services.",
)

s3_vendor: Literal["standard", "aliyun_oss"] = Field(
default="standard",
description="S3 vendor behavior. Use 'aliyun_oss' for Alibaba Cloud OSS.",
)

directory_marker_mode: DirectoryMarkerMode = Field(
default=DirectoryMarkerMode.EMPTY,
description="How to persist S3 directory markers: 'none' skips marker creation, 'empty' writes a zero-byte marker, and 'nonempty' writes a non-empty marker payload. Defaults to 'empty'.",
Expand Down
Loading