Skip to content

WKBCH-15: Update Clickhouse schema #59

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dvasilas merged 1 commit into from
Jan 21, 2026
Merged

WKBCH-15: Update Clickhouse schema #59

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 30 additions & 43 deletions templates/clickhouse/init.d/02-create-ingest-table.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,56 +1,43 @@
CREATE TABLE IF NOT EXISTS logs.access_logs_ingest
(
-- Common
timestamp DateTime,
insertedAt DateTime DEFAULT now(),
hostname LowCardinality(String),

-- Analytics
action LowCardinality(String),
accountName String,
userName String,
clientPort UInt32,
httpMethod LowCardinality(String),
bytesDeleted UInt64,
bytesReceived UInt64,
bodyLength UInt64,
contentLength UInt64,
elapsed_ms Float32,
hostname LowCardinality(Nullable(String)),

-- AWS access server logs fields https://docs.aws.amazon.com/AmazonS3/latest/userguide/LogFormat.html
startTime DateTime64(3), -- AWS "Time" field
requester String,
operation String,
requestURI String,
errorCode String,
objectSize UInt64,
totalTime Float32,
turnAroundTime Float32,
referer String,
userAgent String,
versionId String,
signatureVersion LowCardinality(String),
cipherSuite LowCardinality(String),
authenticationType LowCardinality(String),
hostHeader String,
tlsVersion LowCardinality(String),
aclRequired LowCardinality(String),
requester Nullable(String),
operation Nullable(String),
requestURI Nullable(String),
errorCode Nullable(String),
objectSize Nullable(UInt64),
totalTime Nullable(Float32),
turnAroundTime Nullable(Float32),
referer Nullable(String),
userAgent Nullable(String),
versionId Nullable(String),
signatureVersion LowCardinality(Nullable(String)),
cipherSuite LowCardinality(Nullable(String)),
authenticationType LowCardinality(Nullable(String)),
hostHeader Nullable(String),
tlsVersion LowCardinality(Nullable(String)),
aclRequired LowCardinality(Nullable(String)),

-- Shared between AWS access server logs and Analytics logs
bucketOwner String, -- AWS "Bucket Owner" field
bucketName String, -- AWS "Bucket" field
req_id String, -- AWS "Request ID" field
bytesSent UInt64, -- AWS "Bytes Sent" field
clientIP String, -- AWS "Remote IP" field
httpCode UInt16, -- AWS "HTTP Status" field
objectKey String, -- AWS "Key" field
bucketOwner Nullable(String), -- AWS "Bucket Owner" field
bucketName String DEFAULT '', -- AWS "Bucket" field
req_id String DEFAULT '', -- AWS "Request ID" field
bytesSent Nullable(UInt64), -- AWS "Bytes Sent" field
clientIP Nullable(String), -- AWS "Remote IP" field
httpCode Nullable(UInt16), -- AWS "HTTP Status" field
objectKey Nullable(String), -- AWS "Key" field

-- Scality server access logs extra fields.
logFormatVersion LowCardinality(String),
loggingEnabled Bool,
loggingTargetBucket String,
loggingTargetPrefix String,
awsAccessKeyID String,
raftSessionID UInt16
logFormatVersion LowCardinality(Nullable(String)),
loggingEnabled Bool DEFAULT false,
loggingTargetBucket String DEFAULT '',
loggingTargetPrefix String DEFAULT '',
awsAccessKeyID Nullable(String),
raftSessionID UInt16 DEFAULT 0
)
Engine = Null();
65 changes: 32 additions & 33 deletions templates/clickhouse/init.d/03-create-storage-table.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,46 +1,45 @@
CREATE TABLE IF NOT EXISTS logs.access_logs
(
-- Common
timestamp DateTime,
insertedAt DateTime DEFAULT now(),
hostname LowCardinality(String),
hostname LowCardinality(Nullable(String)),

-- AWS access server logs fields https://docs.aws.amazon.com/AmazonS3/latest/userguide/LogFormat.html
startTime DateTime64(3), -- AWS "Time" field
requester String,
operation String,
requestURI String,
errorCode String,
objectSize UInt64,
totalTime Float32,
turnAroundTime Float32,
referer String,
userAgent String,
versionId String,
signatureVersion LowCardinality(String),
cipherSuite LowCardinality(String),
authenticationType LowCardinality(String),
hostHeader String,
tlsVersion LowCardinality(String),
aclRequired LowCardinality(String),
requester Nullable(String),
operation Nullable(String),
requestURI Nullable(String),
errorCode Nullable(String),
objectSize Nullable(UInt64),
totalTime Nullable(Float32),
turnAroundTime Nullable(Float32),
referer Nullable(String),
userAgent Nullable(String),
versionId Nullable(String),
signatureVersion LowCardinality(Nullable(String)),
cipherSuite LowCardinality(Nullable(String)),
authenticationType LowCardinality(Nullable(String)),
hostHeader Nullable(String),
tlsVersion LowCardinality(Nullable(String)),
aclRequired LowCardinality(Nullable(String)),

-- Shared between AWS access server logs and Analytics logs
bucketOwner String, -- AWS "Bucket Owner" field
bucketName String, -- AWS "Bucket" field
req_id String, -- AWS "Request ID" field
bytesSent UInt64, -- AWS "Bytes Sent" field
clientIP String, -- AWS "Remote IP" field
httpCode UInt16, -- AWS "HTTP Status" field
objectKey String, -- AWS "Key" field
bucketOwner Nullable(String), -- AWS "Bucket Owner" field
bucketName String DEFAULT '', -- AWS "Bucket" field
req_id String DEFAULT '', -- AWS "Request ID" field
bytesSent Nullable(UInt64), -- AWS "Bytes Sent" field
clientIP Nullable(String), -- AWS "Remote IP" field
httpCode Nullable(UInt16), -- AWS "HTTP Status" field
objectKey Nullable(String), -- AWS "Key" field

-- Scality server access logs extra fields.
logFormatVersion LowCardinality(String),
loggingEnabled Bool,
loggingTargetBucket String,
loggingTargetPrefix String,
awsAccessKeyID String,
raftSessionID UInt16
logFormatVersion LowCardinality(Nullable(String)),
loggingEnabled Bool DEFAULT false,
loggingTargetBucket String DEFAULT '',
loggingTargetPrefix String DEFAULT '',
awsAccessKeyID Nullable(String),
raftSessionID UInt16 DEFAULT 0
)
Engine = MergeTree()
PARTITION BY toStartOfInterval(insertedAt, INTERVAL 24 HOUR)
ORDER BY (raftSessionID, bucketName, insertedAt, timestamp, req_id);
PARTITION BY toStartOfInterval(insertedAt, INTERVAL 1 HOUR)
ORDER BY (raftSessionID, bucketName, insertedAt, startTime, req_id);
2 changes: 1 addition & 1 deletion templates/clickhouse/init.d/04-create-offsets-table.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ CREATE TABLE IF NOT EXISTS logs.offsets
bucketName String,
raftSessionID UInt16,
lastProcessedInsertedAt DateTime,
lastProcessedTimestamp DateTime64(3),
lastProcessedStartTime DateTime64(3),
lastProcessedReqId String
)
ENGINE = ReplacingMergeTree(lastProcessedInsertedAt)
Expand Down
1 change: 0 additions & 1 deletion templates/clickhouse/init.d/06-create-materialized-view.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ CREATE MATERIALIZED VIEW IF NOT EXISTS logs.access_logs_ingest_mv
TO logs.access_logs_federated
AS
SELECT
timestamp,
insertedAt,
hostname,

Expand Down
Loading