Skip to content

Commit 8b0c894

Browse files
authored
Merge pull request #854 from sipcapture/feat/s3-url-style-11.0.287
feat: configurable S3 URL style for DuckLake (11.0.287)
2 parents 8428172 + 780ab44 commit 8b0c894

17 files changed

Lines changed: 69 additions & 22 deletions

docs/ENVIRONMENT_VARIABLES.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,9 @@ Field names follow the `mapstructure` tags on [`Config` in `src/config/config.go
1919

2020
- `storage.ducklake.storage_policy.volumes[1].s3_endpoint`
2121
`HOMER_STORAGE_DUCKLAKE_STORAGE_POLICY_VOLUMES_1_S3_ENDPOINT`
22+
- `storage.ducklake.s3.url_style``HOMER_STORAGE_DUCKLAKE_S3_URL_STYLE` (`path` by default; set `vhost` for S3 virtual-hosted-style endpoints)
23+
- `node.ducklake.volumes[0].s3_url_style``HOMER_NODE_DUCKLAKE_VOLUMES_0_S3_URL_STYLE`
24+
- `storage.ducklake.storage_policy.volumes[1].s3_url_style``HOMER_STORAGE_DUCKLAKE_STORAGE_POLICY_VOLUMES_1_S3_URL_STYLE`
2225

2326
## References
2427

docs/NODE.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,7 @@ See [FLIGHTSQL.md](FLIGHTSQL.md) for Grafana setup and coordinator proxy (`coord
148148
| `s3_secret_access_key` | string | - | AWS Secret Access Key |
149149
| `s3_endpoint` | string | - | Custom S3 endpoint (MinIO, R2) |
150150
| `s3_use_ssl` | bool | true | Use SSL for S3 connections |
151+
| `s3_url_style` | string | path | DuckDB S3 URL style for custom endpoints (`path` or `vhost`) |
151152
| `override_data_path` | bool | false | If true, DuckLake `ATTACH` uses `OVERRIDE_DATA_PATH TRUE` so `path` may differ from the `DATA_PATH` already stored in the catalog (e.g. bucket rename). Prefer keeping `path` identical to the writer volume that created the catalog. |
152153

153154
### Volume `name` and the DuckDB catalog

docs/STORAGE_POLICIES.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,7 @@ The `move_factor` parameter works similar to ClickHouse storage policies. It con
122122
| `s3_secret_access_key` | string | "" | Secret key |
123123
| `s3_endpoint` | string | "" | Custom endpoint for S3-compatible services (R2, MinIO, RustFS) |
124124
| `s3_use_ssl` | bool | true | Use HTTPS for S3 connections |
125+
| `s3_url_style` | string | "path" | DuckDB S3 URL style for custom endpoints (`path` or `vhost`) |
125126

126127
## Examples
127128

src/cli/cli_cmd.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,7 @@ func duckLakeConfigFromModular(cfg *config.Config) ducklake.Config {
110110
base.S3SecretAccessKey = source.S3.SecretAccessKey
111111
base.S3Endpoint = source.S3.Endpoint
112112
base.S3UseSSL = source.S3.UseSSL
113+
base.S3URLStyle = source.S3.URLStyle
113114
}
114115

115116
return base
@@ -124,14 +125,14 @@ func openDuckLakeReadOnly(cfg ducklake.Config) (*sql.DB, error) {
124125
}
125126

126127
if err := ducklake.ApplyDuckDBS3ClientSettings(db,
127-
cfg.S3Region, cfg.S3AccessKeyID, cfg.S3SecretAccessKey, cfg.S3Endpoint, cfg.S3UseSSL,
128+
cfg.S3Region, cfg.S3AccessKeyID, cfg.S3SecretAccessKey, cfg.S3Endpoint, cfg.S3UseSSL, cfg.S3URLStyle,
128129
); err != nil {
129130
db.Close()
130131
return nil, fmt.Errorf("failed to configure S3: %w", err)
131132
}
132133
if ducklake.IsRemoteLakeDataPath(cfg.DataPath) {
133134
if err := ducklake.EnsureWriterS3Secret(db,
134-
cfg.S3Region, cfg.S3AccessKeyID, cfg.S3SecretAccessKey, cfg.S3Endpoint, cfg.S3UseSSL,
135+
cfg.S3Region, cfg.S3AccessKeyID, cfg.S3SecretAccessKey, cfg.S3Endpoint, cfg.S3UseSSL, cfg.S3URLStyle,
135136
); err != nil {
136137
db.Close()
137138
return nil, fmt.Errorf("failed to configure S3 secret: %w", err)

src/config/config.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -737,6 +737,7 @@ type VolumeConfig struct {
737737
S3SecretKey string `json:"s3_secret_access_key" mapstructure:"s3_secret_access_key" default:""`
738738
S3Endpoint string `json:"s3_endpoint" mapstructure:"s3_endpoint" default:""` // For S3-compatible (R2, MinIO)
739739
S3UseSSL bool `json:"s3_use_ssl" mapstructure:"s3_use_ssl" default:"true"`
740+
S3URLStyle string `json:"s3_url_style" mapstructure:"s3_url_style" default:""` // Empty = path, set "vhost" for virtual-hosted-style
740741
// OverrideDataPath passes OVERRIDE_DATA_PATH TRUE to DuckLake ATTACH when the path
741742
// in config intentionally differs from DATA_PATH stored in an existing catalog
742743
// (e.g. bucket rename, or node path typo vs writer). Prefer matching paths first.
@@ -792,6 +793,7 @@ type S3Config struct {
792793
SecretAccessKey string `json:"secret_access_key" mapstructure:"secret_access_key" default:""`
793794
Endpoint string `json:"endpoint" mapstructure:"endpoint" default:""`
794795
UseSSL bool `json:"use_ssl" mapstructure:"use_ssl" default:"true"`
796+
URLStyle string `json:"url_style" mapstructure:"url_style" default:""` // Empty = path, set "vhost" for virtual-hosted-style
795797
}
796798

797799
// HEPConfig configures HEP protocol processing

src/homerconfig/dataconfig.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -275,6 +275,7 @@ type HomerServerSettings struct {
275275
SecretAccessKey string `json:"secret_access_key" mapstructure:"secret_access_key" default:""`
276276
Endpoint string `json:"endpoint" mapstructure:"endpoint" default:""`
277277
UseSSL bool `json:"use_ssl" mapstructure:"use_ssl" default:"true"`
278+
URLStyle string `json:"url_style" mapstructure:"url_style" default:""`
278279
} `json:"s3" mapstructure:"s3"`
279280
} `json:"ducklake_settings" mapstructure:"ducklake_settings"`
280281
}

src/main.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -353,6 +353,7 @@ func ducklakeConfigFromModular(cfg *config.Config) ducklake.Config {
353353
base.S3SecretAccessKey = source.S3.SecretAccessKey
354354
base.S3Endpoint = source.S3.Endpoint
355355
base.S3UseSSL = source.S3.UseSSL
356+
base.S3URLStyle = source.S3.URLStyle
356357
}
357358

358359
return base

src/node/node.go

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1507,6 +1507,7 @@ func configureDuckLake(db *sql.DB, cfg *config.NodeConfig) ([]VolumeInfo, error)
15071507
cfg.DuckLake.S3.SecretAccessKey,
15081508
cfg.DuckLake.S3.Endpoint,
15091509
cfg.DuckLake.S3.UseSSL,
1510+
cfg.DuckLake.S3.URLStyle,
15101511
); err != nil {
15111512
return nil, fmt.Errorf("failed to configure S3: %w", err)
15121513
}
@@ -1576,6 +1577,7 @@ func attachVolume(db *sql.DB, baseLakeName string, vol config.VolumeConfig) (Vol
15761577
if region == "" && endpoint != "" {
15771578
region = "us-east-1"
15781579
}
1580+
urlStyle := strings.ReplaceAll(ducklake.NormalizeS3URLStyle(vol.S3URLStyle), "'", "''")
15791581

15801582
// Create secret
15811583
var createSecret string
@@ -1587,10 +1589,10 @@ func attachVolume(db *sql.DB, baseLakeName string, vol config.VolumeConfig) (Vol
15871589
SECRET '%s',
15881590
REGION '%s',
15891591
ENDPOINT '%s',
1590-
URL_STYLE 'path',
1592+
URL_STYLE '%s',
15911593
USE_SSL %t
15921594
);
1593-
`, secretName, vol.S3AccessKeyID, vol.S3SecretKey, region, endpoint, vol.S3UseSSL)
1595+
`, secretName, vol.S3AccessKeyID, vol.S3SecretKey, region, endpoint, urlStyle, vol.S3UseSSL)
15941596
} else {
15951597
createSecret = fmt.Sprintf(`
15961598
CREATE SECRET %s (

src/storage/ducklake/ducklake.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,7 @@ type Config struct {
8686
S3SecretAccessKey string
8787
S3Endpoint string
8888
S3UseSSL bool
89+
S3URLStyle string
8990

9091
// DuckDB engine tuning. Empty / zero values mean "leave DuckDB's
9192
// own default" — these knobs are opt-in. See ApplyDuckDBTuning
@@ -345,6 +346,7 @@ func (mtw *MultiTableWriter) connect() error {
345346
mtw.config.S3SecretAccessKey,
346347
mtw.config.S3Endpoint,
347348
mtw.config.S3UseSSL,
349+
mtw.config.S3URLStyle,
348350
)
349351
connector, err := duckdb.NewConnector("", func(execer driver.ExecerContext) error {
350352
for _, stmt := range s3Stmts {
@@ -482,6 +484,7 @@ func (mtw *MultiTableWriter) connect() error {
482484
mtw.config.S3SecretAccessKey,
483485
mtw.config.S3Endpoint,
484486
mtw.config.S3UseSSL,
487+
mtw.config.S3URLStyle,
485488
); err != nil {
486489
return fmt.Errorf("failed to configure S3 secret for DuckLake: %w", err)
487490
}

src/storage/ducklake/manager.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@ func NewManagerFromConfig() (*Manager, error) {
7272
config.S3SecretAccessKey = settings.S3.SecretAccessKey
7373
config.S3Endpoint = settings.S3.Endpoint
7474
config.S3UseSSL = settings.S3.UseSSL
75+
config.S3URLStyle = settings.S3.URLStyle
7576
}
7677

7778
// Apply defaults

0 commit comments

Comments
 (0)