Skip to content

Commit 8b73e20

Browse files
luluz66luluz66
committed
add new codec change and partition by to Clickhouse Executions table
1 parent 131841b commit 8b73e20

1 file changed

Lines changed: 86 additions & 84 deletions

File tree

‎server/util/clickhouse/schema/schema.go‎

Lines changed: 86 additions & 84 deletions
Original file line numberDiff line numberDiff line change
@@ -144,12 +144,13 @@ func (i *Invocation) TableOptions(clickhouseVersion string) string {
144144

145145
type Execution struct {
146146
// Sort keys
147-
GroupID string
148-
UpdatedAtUsec int64
149-
InvocationUUID string
150-
ExecutionID string
147+
GroupID string `gorm:"codec:ZSTD(1)"`
148+
UpdatedAtUsec int64 `gorm:"codec:DoubleDelta,ZSTD(1)"`
149+
InvocationUUID string `gorm:"codec:ZSTD(1)"`
150+
ExecutionID string `gorm:"codec:ZSTD(1)"`
151151

152152
// Resource-name components split out of execution_id.
153+
// ActionDigest holds raw hash bytes (incompressible) — default codec.
153154
InstanceName string `gorm:"codec:ZSTD(1)"`
154155
ExecutionUUID string `gorm:"type:UUID"`
155156
Compressor string `gorm:"type:LowCardinality(String)"`
@@ -158,126 +159,127 @@ type Execution struct {
158159
ActionDigestSize uint32 `gorm:"codec:T64,ZSTD(1)"`
159160

160161
// Type from tables.InvocationExecution
161-
InvocationLinkType int8
162-
CreatedAtUsec int64
163-
UserID string
164-
Worker string
165-
ExecutorHostname string
166-
ClientIP string
162+
InvocationLinkType int8 `gorm:"codec:T64,ZSTD(1)"`
163+
CreatedAtUsec int64 `gorm:"codec:DoubleDelta,ZSTD(1)"`
164+
UserID string `gorm:"codec:ZSTD(1)"`
165+
Worker string `gorm:"codec:ZSTD(1)"`
166+
ExecutorHostname string `gorm:"codec:ZSTD(1)"`
167+
ClientIP string `gorm:"codec:ZSTD(1)"`
167168

168169
// Executor metadata
169170
SelfHosted bool
170171
Region string `gorm:"type:LowCardinality(String)"`
171172

172-
Stage int64
173+
Stage int64 `gorm:"codec:T64,ZSTD(1)"`
173174

174175
// RequestMetadata
175-
TargetLabel string
176-
ActionMnemonic string
176+
TargetLabel string `gorm:"codec:ZSTD(1)"`
177+
ActionMnemonic string `gorm:"codec:ZSTD(1)"`
177178

178179
// IOStats
179-
FileDownloadCount int64
180-
FileDownloadSizeBytes int64
181-
FileDownloadDurationUsec int64
182-
FileUploadCount int64
183-
FileUploadSizeBytes int64
184-
FileUploadDurationUsec int64
180+
FileDownloadCount int64 `gorm:"codec:T64,ZSTD(1)"`
181+
FileDownloadSizeBytes int64 `gorm:"codec:T64,ZSTD(1)"`
182+
FileDownloadDurationUsec int64 `gorm:"codec:T64,ZSTD(1)"`
183+
FileUploadCount int64 `gorm:"codec:T64,ZSTD(1)"`
184+
FileUploadSizeBytes int64 `gorm:"codec:T64,ZSTD(1)"`
185+
FileUploadDurationUsec int64 `gorm:"codec:T64,ZSTD(1)"`
185186

186187
// UsageStats
187-
PeakMemoryBytes int64
188-
CPUNanos int64
189-
DiskBytesRead int64
190-
DiskBytesWritten int64
191-
DiskReadOperations int64
192-
DiskWriteOperations int64
193-
NetworkBytesSent int64
194-
NetworkBytesReceived int64
195-
NetworkPacketsSent int64
196-
NetworkPacketsReceived int64
188+
PeakMemoryBytes int64 `gorm:"codec:T64,ZSTD(1)"`
189+
CPUNanos int64 `gorm:"codec:T64,ZSTD(1)"`
190+
DiskBytesRead int64 `gorm:"codec:T64,ZSTD(1)"`
191+
DiskBytesWritten int64 `gorm:"codec:T64,ZSTD(1)"`
192+
DiskReadOperations int64 `gorm:"codec:T64,ZSTD(1)"`
193+
DiskWriteOperations int64 `gorm:"codec:T64,ZSTD(1)"`
194+
NetworkBytesSent int64 `gorm:"codec:T64,ZSTD(1)"`
195+
NetworkBytesReceived int64 `gorm:"codec:T64,ZSTD(1)"`
196+
NetworkPacketsSent int64 `gorm:"codec:T64,ZSTD(1)"`
197+
NetworkPacketsReceived int64 `gorm:"codec:T64,ZSTD(1)"`
197198

198199
// UsageStats - Linux PSI stats
199-
CPUPressureSomeStallUsec int64
200-
CPUPressureFullStallUsec int64
201-
MemoryPressureSomeStallUsec int64
202-
MemoryPressureFullStallUsec int64
203-
IOPressureSomeStallUsec int64
204-
IOPressureFullStallUsec int64
200+
CPUPressureSomeStallUsec int64 `gorm:"codec:T64,ZSTD(1)"`
201+
CPUPressureFullStallUsec int64 `gorm:"codec:T64,ZSTD(1)"`
202+
MemoryPressureSomeStallUsec int64 `gorm:"codec:T64,ZSTD(1)"`
203+
MemoryPressureFullStallUsec int64 `gorm:"codec:T64,ZSTD(1)"`
204+
IOPressureSomeStallUsec int64 `gorm:"codec:T64,ZSTD(1)"`
205+
IOPressureFullStallUsec int64 `gorm:"codec:T64,ZSTD(1)"`
205206

206207
// Task sizing
207-
EstimatedMemoryBytes int64
208-
EstimatedMilliCPU int64
209-
EstimatedFreeDiskBytes int64
210-
RequestedComputeUnits float64
211-
RequestedMemoryBytes int64
212-
RequestedMilliCPU int64
213-
RequestedFreeDiskBytes int64
214-
PreviousMeasuredMemoryBytes int64
215-
PreviousMeasuredMilliCPU int64
216-
PreviousMeasuredFreeDiskBytes int64
217-
PredictedMemoryBytes int64
218-
PredictedMilliCPU int64
219-
PredictedFreeDiskBytes int64
208+
EstimatedMemoryBytes int64 `gorm:"codec:T64,ZSTD(1)"`
209+
EstimatedMilliCPU int64 `gorm:"codec:T64,ZSTD(1)"`
210+
EstimatedFreeDiskBytes int64 `gorm:"codec:T64,ZSTD(1)"`
211+
RequestedComputeUnits float64 `gorm:"codec:ZSTD(1)"`
212+
RequestedMemoryBytes int64 `gorm:"codec:T64,ZSTD(1)"`
213+
RequestedMilliCPU int64 `gorm:"codec:T64,ZSTD(1)"`
214+
RequestedFreeDiskBytes int64 `gorm:"codec:T64,ZSTD(1)"`
215+
PreviousMeasuredMemoryBytes int64 `gorm:"codec:T64,ZSTD(1)"`
216+
PreviousMeasuredMilliCPU int64 `gorm:"codec:T64,ZSTD(1)"`
217+
PreviousMeasuredFreeDiskBytes int64 `gorm:"codec:T64,ZSTD(1)"`
218+
PredictedMemoryBytes int64 `gorm:"codec:T64,ZSTD(1)"`
219+
PredictedMilliCPU int64 `gorm:"codec:T64,ZSTD(1)"`
220+
PredictedFreeDiskBytes int64 `gorm:"codec:T64,ZSTD(1)"`
220221

221222
// ExecutedActionMetadata (in addition to Worker above)
222-
QueuedTimestampUsec int64
223-
WorkerStartTimestampUsec int64
224-
WorkerCompletedTimestampUsec int64
225-
InputFetchStartTimestampUsec int64
226-
InputFetchCompletedTimestampUsec int64
227-
ExecutionStartTimestampUsec int64
228-
ExecutionCompletedTimestampUsec int64
229-
OutputUploadStartTimestampUsec int64
230-
OutputUploadCompletedTimestampUsec int64
231-
232-
StatusCode int32
233-
ExitCode int32
223+
QueuedTimestampUsec int64 `gorm:"codec:DoubleDelta,ZSTD(1)"`
224+
WorkerStartTimestampUsec int64 `gorm:"codec:DoubleDelta,ZSTD(1)"`
225+
WorkerCompletedTimestampUsec int64 `gorm:"codec:DoubleDelta,ZSTD(1)"`
226+
InputFetchStartTimestampUsec int64 `gorm:"codec:DoubleDelta,ZSTD(1)"`
227+
InputFetchCompletedTimestampUsec int64 `gorm:"codec:DoubleDelta,ZSTD(1)"`
228+
ExecutionStartTimestampUsec int64 `gorm:"codec:DoubleDelta,ZSTD(1)"`
229+
ExecutionCompletedTimestampUsec int64 `gorm:"codec:DoubleDelta,ZSTD(1)"`
230+
OutputUploadStartTimestampUsec int64 `gorm:"codec:DoubleDelta,ZSTD(1)"`
231+
OutputUploadCompletedTimestampUsec int64 `gorm:"codec:DoubleDelta,ZSTD(1)"`
232+
233+
StatusCode int32 `gorm:"codec:T64,ZSTD(1)"`
234+
ExitCode int32 `gorm:"codec:T64,ZSTD(1)"`
234235

235236
CachedResult bool
236237
DoNotCache bool
237238
SkipCacheLookup bool
238239

239-
ExecutionPriority int32
240-
RequestedIsolationType string
240+
ExecutionPriority int32 `gorm:"codec:T64,ZSTD(1)"`
241+
RequestedIsolationType string `gorm:"type:LowCardinality(String)"`
241242
EffectiveIsolationType string `gorm:"type:LowCardinality(String)"` // This values comes from the executor
242-
RequestedPool string
243-
EffectivePool string
243+
RequestedPool string `gorm:"codec:ZSTD(1)"`
244+
EffectivePool string `gorm:"codec:ZSTD(1)"`
244245

245246
// Runner metadata
246-
RunnerID string
247-
RunnerTaskNumber int64
248-
PlatformHash string
249-
PersistentWorkerKey string
247+
RunnerID string `gorm:"codec:ZSTD(1)"`
248+
RunnerTaskNumber int64 `gorm:"codec:T64,ZSTD(1)"`
249+
PlatformHash string `gorm:"codec:ZSTD(1)"`
250+
PersistentWorkerKey string `gorm:"codec:ZSTD(1)"`
250251

251-
RequestedTimeoutUsec int64
252-
EffectiveTimeoutUsec int64
252+
RequestedTimeoutUsec int64 `gorm:"codec:T64,ZSTD(1)"`
253+
EffectiveTimeoutUsec int64 `gorm:"codec:T64,ZSTD(1)"`
253254

254255
Experiments []string `gorm:"type:Array(LowCardinality(String))"`
255256

256257
// Long string fields
257-
OutputPath string
258-
StatusMessage string
259-
CommandSnippet string
258+
OutputPath string `gorm:"codec:ZSTD(1)"`
259+
StatusMessage string `gorm:"codec:ZSTD(1)"`
260+
CommandSnippet string `gorm:"codec:ZSTD(1)"`
260261

261262
// Fields from Invocations
262-
User string
263-
Host string
264-
Pattern string
265-
Role string
266-
BranchName string
267-
CommitSHA string
268-
RepoURL string
269-
Command string
270-
InvocationStatus int64
263+
User string `gorm:"codec:ZSTD(1)"`
264+
Host string `gorm:"codec:ZSTD(1)"`
265+
Pattern string `gorm:"codec:ZSTD(1)"`
266+
Role string `gorm:"type:LowCardinality(String)"`
267+
BranchName string `gorm:"codec:ZSTD(1)"`
268+
CommitSHA string `gorm:"codec:ZSTD(1)"`
269+
RepoURL string `gorm:"codec:ZSTD(1)"`
270+
Command string `gorm:"codec:ZSTD(1)"`
271+
InvocationStatus int64 `gorm:"codec:T64,ZSTD(1)"`
271272
Success bool
272-
Tags []string `gorm:"type:Array(String);"`
273+
Tags []string `gorm:"type:Array(String);codec:ZSTD(1)"`
273274
}
274275

275276
func (e *Execution) TableName() string {
276277
return "Executions"
277278
}
278279

279280
func (e *Execution) TableOptions(clickhouseVersion string) string {
280-
return fmt.Sprintf("ENGINE=%s ORDER BY (group_id, updated_at_usec, invocation_uuid,execution_id)", getEngine())
281+
return fmt.Sprintf("ENGINE=%s ORDER BY (group_id, updated_at_usec, invocation_uuid,execution_id)", getEngine()) +
282+
" PARTITION BY toYYYYMM(toDateTime(intDiv(updated_at_usec, 1000000), 'UTC'))"
281283
}
282284

283285
func (e *Execution) ExcludedFields() []string {

0 commit comments

Comments
 (0)