Skip to content

Commit 829424e

Browse files
dvasilasdvasilas
authored
Merge pull request #54 from scality/improvement/WKBCH-14
WKBCH-14: Add cloudserver server-access.log and FluentBit for bucket logging
2 parents 141b02f + 71d1f01 commit 829424e

File tree

10 files changed

+165
-2
lines changed

10 files changed

+165
-2
lines changed

‎.github/dependabot.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,8 @@ updates:
2626
update-types: ["version-update:semver-major"]
2727
- dependency-name: "redis"
2828
update-types: ["version-update:semver-major"]
29+
- dependency-name: "clickhouse/clickhouse-server"
30+
- dependency-name: "fluent/fluent-bit"
2931
groups:
3032
docker:
3133
patterns:

‎cmd/config.go‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ type EnvironmentConfig struct {
5858
Utapi UtapiConfig `yaml:"utapi"`
5959
MigrationTools MigrationToolsConfig `yaml:"migration_tools"`
6060
Clickhouse ClickhouseConfig `yaml:"clickhouse"`
61+
Fluentbit FluentbitConfig `yaml:"fluentbit"`
6162
}
6263

6364
type GlobalConfig struct {
@@ -219,6 +220,11 @@ type ClickhouseConfig struct {
219220
LogLevel string `yaml:"log_level"`
220221
}
221222

223+
type FluentbitConfig struct {
224+
Image string `yaml:"image"`
225+
LogLevel string `yaml:"log_level"`
226+
}
227+
222228
type AccessLoggingFeatureConfig struct {
223229
Enabled bool `yaml:"enabled"`
224230
}
@@ -286,7 +292,8 @@ func DefaultEnvironmentConfig() EnvironmentConfig {
286292
},
287293
Utapi: UtapiConfig{},
288294
MigrationTools: MigrationToolsConfig{},
289-
Clickhouse: ClickhouseConfig{},
295+
Clickhouse: ClickhouseConfig{},
296+
Fluentbit: FluentbitConfig{},
290297
}
291298
}
292299

@@ -367,5 +374,9 @@ func LoadEnvironmentConfig(path string) (EnvironmentConfig, error) {
367374
cfg.Clickhouse.LogLevel = cfg.Global.LogLevel
368375
}
369376

377+
if cfg.Fluentbit.LogLevel == "" {
378+
cfg.Fluentbit.LogLevel = cfg.Global.LogLevel
379+
}
380+
370381
return cfg, nil
371382
}

‎cmd/configure.go‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,11 +37,13 @@ func (c *ConfigureCmd) Run() error {
3737
func createLogDirectories(envDir string) error {
3838
logDirs := []string{
3939
filepath.Join(envDir, "logs"),
40+
filepath.Join(envDir, "logs", "cloudserver"),
4041
filepath.Join(envDir, "logs", "scuba"),
4142
filepath.Join(envDir, "logs", "backbeat"),
4243
filepath.Join(envDir, "logs", "migration-tools"),
4344
filepath.Join(envDir, "logs", "clickhouse-shard-1"),
4445
filepath.Join(envDir, "logs", "clickhouse-shard-2"),
46+
filepath.Join(envDir, "logs", "fluentbit"),
4547
}
4648

4749
for _, dir := range logDirs {
@@ -74,6 +76,7 @@ func configureEnv(cfg EnvironmentConfig, envDir string) error {
7476
generateUtapiConfig,
7577
generateMigrationToolsConfig,
7678
generateClickhouseConfig,
79+
generateFluentbitConfig,
7780
}
7881

7982
configDir := filepath.Join(envDir, "config")
@@ -222,3 +225,12 @@ func generateClickhouseConfig(cfg EnvironmentConfig, path string) error {
222225

223226
return renderTemplates(cfg, "templates/clickhouse", filepath.Join(path, "clickhouse"), templates)
224227
}
228+
229+
func generateFluentbitConfig(cfg EnvironmentConfig, path string) error {
230+
templates := []string{
231+
"fluent-bit.conf",
232+
"parsers.conf",
233+
}
234+
235+
return renderTemplates(cfg, "templates/fluentbit", filepath.Join(path, "fluentbit"), templates)
236+
}

‎templates/clickhouse/init.d/02-create-ingest-table.sql‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@ CREATE TABLE IF NOT EXISTS logs.access_logs_ingest
88
-- Analytics
99
action LowCardinality(String),
1010
accountName String,
11-
accountDisplayName String,
1211
userName String,
1312
clientPort UInt32,
1413
httpMethod LowCardinality(String),

‎templates/cloudserver/config-v9.json‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -177,5 +177,16 @@
177177
"integrityChecks": {
178178
"objectPutRetention": true
179179
},
180+
"serverAccessLogs": {
181+
{{- if .Features.AccessLogging.Enabled }}
182+
"mode": "ENABLED",
183+
{{- else }}
184+
"mode": "DISABLED",
185+
{{- end }}
186+
"outputFile": "/logs/server-access.log",
187+
"highWaterMarkBytes": 10485760,
188+
"retryReopenDelayMS": 1000,
189+
"checkFileRotationIntervalMS": 10000
190+
},
180191
"testingMode": true
181192
}

‎templates/fluentbit/fluent-bit.conf‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
[SERVICE]
2+
Flush 10
3+
Grace 10
4+
Parsers_File /fluent-bit/etc/parsers.conf
5+
6+
Log_Level {{ .Fluentbit.LogLevel }}
7+
Log_File /var/log/fluent-bit/fluent-bit.log
8+
9+
# HTTP server for prometheus metrics
10+
HTTP_Server On
11+
HTTP_Listen 0.0.0.0
12+
HTTP_PORT 2020
13+
14+
[INPUT]
15+
Name tail
16+
Path /fluent-bit/log/server-access.log
17+
Tag access_logging
18+
Read_from_Head true
19+
20+
# Sync with cloudserver checkFileRotationIntervalMS (10000ms)
21+
# rotate_wait should be >= checkFileRotationIntervalMS to avoid losing logs
22+
rotate_wait 30
23+
refresh_interval 5
24+
25+
# Database for tracking file offsets across restarts
26+
DB /fluent-bit/data/tail.db
27+
DB.locking true
28+
DB.sync normal
29+
DB.journal_mode wal
30+
31+
[FILTER]
32+
Name parser
33+
Key_Name log
34+
Match access_logging
35+
Parser json_parser
36+
37+
[FILTER]
38+
Name record_modifier
39+
Match access_logging
40+
41+
# Common
42+
Allowlist_key hostname
43+
44+
# Analytics
45+
Allowlist_key action
46+
Allowlist_key accountName
47+
Allowlist_key httpMethod
48+
Allowlist_key userName
49+
Allowlist_key bytesDeleted
50+
Allowlist_key bytesReceived
51+
Allowlist_key bodyLength
52+
Allowlist_key contentLength
53+
Allowlist_key elapsed_ms
54+
55+
# AWS
56+
Allowlist_key startTime
57+
Allowlist_key requester
58+
Allowlist_key operation
59+
Allowlist_key requestURI
60+
Allowlist_key errorCode
61+
Allowlist_key objectSize
62+
Allowlist_key totalTime
63+
Allowlist_key turnAroundTime
64+
Allowlist_key referer
65+
Allowlist_key userAgent
66+
Allowlist_key versionId
67+
Allowlist_key signatureVersion
68+
Allowlist_key cipherSuite
69+
Allowlist_key authenticationType
70+
Allowlist_key hostHeader
71+
Allowlist_key tlsVersion
72+
Allowlist_key aclRequired
73+
74+
# Shared AWS and Analytics
75+
Allowlist_key bucketOwner
76+
Allowlist_key bucketName
77+
Allowlist_key req_id
78+
Allowlist_key bytesSent
79+
Allowlist_key clientIP
80+
Allowlist_key httpCode
81+
Allowlist_key objectKey
82+
83+
# Scality server access logs extra fields
84+
Allowlist_key logFormatVersion
85+
Allowlist_key loggingEnabled
86+
Allowlist_key loggingTargetBucket
87+
Allowlist_key loggingTargetPrefix
88+
Allowlist_key awsAccessKeyID
89+
Allowlist_key raftSessionID
90+
91+
[OUTPUT]
92+
Name http
93+
Match access_logging
94+
Host 127.0.0.1
95+
Port 8123
96+
URI /?query=INSERT+INTO+logs.access_logs_ingest+FORMAT+JSONEachRow
97+
format json_stream
98+
json_date_key timestamp
99+
json_date_format epoch
100+
Retry_Limit 5

‎templates/fluentbit/parsers.conf‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
[PARSER]
2+
Name json_parser
3+
Format json
4+
Time_Key time
5+
Time_Format %s.%L

‎templates/global/defaults.env‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ BACKBEAT_IMAGE="{{ .Backbeat.Image }}"
1010
UTAPI_IMAGE="{{ .Utapi.Image }}"
1111
MIGRATION_TOOLS_IMAGE="{{ .MigrationTools.Image }}"
1212
CLICKHOUSE_IMAGE="{{ .Clickhouse.Image }}"
13+
FLUENTBIT_IMAGE="{{ .Fluentbit.Image }}"
1314

1415
METADATA_S3_DB_VERSION="{{ .S3Metadata.VFormat }}"
1516
CLOUDSERVER_ENABLE_NULL_VERSION_COMPAT_MODE="{{ .Cloudserver.EnableNullVersionCompatMode }}"

‎templates/global/docker-compose.yaml‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,9 +66,28 @@ services:
6666
volumes:
6767
- ./config/cloudserver/config.json:/conf/config.json:ro
6868
- ./config/cloudserver/locationConfig.json:/conf/locationConfig.json:ro
69+
- ./logs/cloudserver:/logs:rw
6970
profiles:
7071
- base
7172

73+
fluentbit:
74+
image: ${FLUENTBIT_IMAGE}
75+
container_name: workbench-fluentbit
76+
network_mode: host
77+
depends_on:
78+
clickhouse-shard-1:
79+
condition: service_healthy
80+
clickhouse-shard-2:
81+
condition: service_healthy
82+
volumes:
83+
- ./config/fluentbit/fluent-bit.conf:/fluent-bit/etc/fluent-bit.conf:ro
84+
- ./config/fluentbit/parsers.conf:/fluent-bit/etc/parsers.conf:ro
85+
- ./logs/cloudserver:/fluent-bit/log:ro
86+
- ./logs/fluentbit:/var/log/fluent-bit:rw
87+
- ./data/fluentbit:/fluent-bit/data:rw
88+
profiles:
89+
- feature-access-logging
90+
7291
vault:
7392
image: ${VAULT_IMAGE}
7493
container_name: workbench-vault

‎templates/global/values.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,3 +60,6 @@ migration_tools:
6060

6161
clickhouse:
6262
image: clickhouse/clickhouse-server:24.3.2.23-alpine
63+
64+
fluentbit:
65+
image: fluent/fluent-bit:3.2.2

0 commit comments

Comments
 (0)