Skip to content

Commit b8915ef

Browse files
zhengkezhou1zhengkezhou1
committed
[feat][storage]: add write path for ClickHouse based on Jaeger V2
Signed-off-by: zzzk1 <[email protected]>
1 parent 4b884bb commit b8915ef

24 files changed

+1214
-3
lines changed

‎.github/workflows/ci-e2e-all.yml

+2
Original file line numberDiff line numberDiff line change
@@ -37,3 +37,5 @@ jobs:
3737
opensearch:
3838
uses: ./.github/workflows/ci-e2e-opensearch.yml
3939

40+
clickhouse:
41+
uses: ./.github/workflows/ci-e2e-clickhouse.yml

‎.github/workflows/ci-e2e-clickhouse.yml

+49
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
name: CIT ClickHouse
2+
3+
on:
4+
workflow_call:
5+
6+
concurrency:
7+
group: cit-kafka-${{ github.workflow }}-${{ (github.event.pull_request && github.event.pull_request.number) || github.ref || github.run_id }}
8+
cancel-in-progress: true
9+
10+
# See https://github.com/ossf/scorecard/blob/main/docs/checks.md#token-permissions
11+
permissions: # added using https://github.com/step-security/secure-workflows
12+
contents: read
13+
14+
jobs:
15+
clickhouse:
16+
runs-on: ubuntu-latest
17+
strategy:
18+
fail-fast: false
19+
matrix:
20+
jaeger-version: [v2]
21+
clickhouse-version: ["25.x"]
22+
name: clickhouse ${{matrix.clickhouse-version }} ${{ matrix.jaeger-version }}
23+
steps:
24+
- name: Harden Runner
25+
uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1
26+
with:
27+
egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs
28+
29+
- uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
30+
31+
- uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0
32+
with:
33+
go-version: 1.24.x
34+
35+
- name: Run clickhouse integration tests
36+
id: test-execution
37+
run: bash scripts/e2e/clickhouse.sh
38+
39+
- uses: ./.github/actions/verify-metrics-snapshot
40+
if: matrix.jaeger-version == 'v2'
41+
with:
42+
snapshot: metrics_snapshot_clickhouse
43+
artifact_key: metrics_snapshot_clickhouse_${{ matrix.jaeger-version }}
44+
45+
- name: Upload coverage to codecov
46+
uses: ./.github/actions/upload-codecov
47+
with:
48+
files: cover.out
49+
flags: clickhouse-${{ matrix.clickhouse-version }}-${{ matrix.jaeger-version }}

‎.mockery.yaml

+5
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,11 @@ packages:
4242
github.com/jaegertracing/jaeger/pkg/es/client:
4343
config:
4444
all: true
45+
github.com/jaegertracing/jaeger/pkg/clickhouse:
46+
config:
47+
all: true
48+
interfaces:
49+
Client:
4550
github.com/jaegertracing/jaeger/pkg/kafka/consumer:
4651
interfaces:
4752
Consumer:

‎docker-compose/clickhouse/docker-compose.yml

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
services:
2+
clickhouse:
3+
container_name: clickhouse
4+
image: bitnami/clickhouse:25.1.3
5+
environment:
6+
CLICKHOUSE_USER: "default"
7+
CLICKHOUSE_PASSWORD: "default"
8+
ports:
9+
- 9000:9000
10+
volumes:
11+
- ./init.sql:/docker-entrypoint-initdb.d/init.sql

‎docker-compose/clickhouse/init.sql

+48
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
CREATE DATABASE IF NOT EXISTS jaeger;
2+
3+
CREATE TABLE IF NOT EXISTS jaeger.otel_traces (
4+
Timestamp DateTime64(9) CODEC(Delta, ZSTD(1)),
5+
TraceId String CODEC(ZSTD(1)),
6+
SpanId String CODEC(ZSTD(1)),
7+
ParentSpanId String CODEC(ZSTD(1)),
8+
TraceState String CODEC(ZSTD(1)),
9+
SpanName LowCardinality(String) CODEC(ZSTD(1)),
10+
SpanKind LowCardinality(String) CODEC(ZSTD(1)),
11+
ServiceName LowCardinality(String) CODEC(ZSTD(1)),
12+
ResourceAttributes Nested
13+
(
14+
keys LowCardinality(String),
15+
values String
16+
) CODEC (ZSTD(1)),
17+
ScopeName String CODEC(ZSTD(1)),
18+
ScopeVersion String CODEC(ZSTD(1)),
19+
SpanAttributes Nested
20+
(
21+
keys LowCardinality(String),
22+
values String
23+
) CODEC (ZSTD(1)),
24+
Duration UInt64 CODEC(ZSTD(1)),
25+
StatusCode LowCardinality(String) CODEC(ZSTD(1)),
26+
StatusMessage String CODEC(ZSTD(1)),
27+
Events Nested (
28+
Timestamp DateTime64(9),
29+
Name LowCardinality(String),
30+
Attributes Map(LowCardinality(String), String)
31+
) CODEC(ZSTD(1)),
32+
Links Nested (
33+
TraceId String,
34+
SpanId String,
35+
TraceState String,
36+
Attributes Map(LowCardinality(String), String)
37+
) CODEC(ZSTD(1)),
38+
INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1,
39+
INDEX idx_res_attr_key ResourceAttributes.keys TYPE bloom_filter(0.01) GRANULARITY 1,
40+
INDEX idx_res_attr_value ResourceAttributes.values TYPE bloom_filter(0.01) GRANULARITY 1,
41+
INDEX idx_span_attr_key SpanAttributes.keys TYPE bloom_filter(0.01) GRANULARITY 1,
42+
INDEX idx_span_attr_value SpanAttributes.values TYPE bloom_filter(0.01) GRANULARITY 1,
43+
INDEX idx_duration Duration TYPE minmax GRANULARITY 1
44+
) ENGINE MergeTree()
45+
46+
PARTITION BY toDate(Timestamp)
47+
ORDER BY (ServiceName, SpanName, toUnixTimestamp(Timestamp), SpanAttributes.keys, SpanAttributes.values, Duration, TraceId)
48+
SETTINGS index_granularity=8192, ttl_only_drop_parts = 1;

‎go.mod

+15-1
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ go 1.23.7
55
toolchain go1.24.0
66

77
require (
8+
github.com/ClickHouse/ch-go v0.64.1
9+
github.com/ClickHouse/clickhouse-go/v2 v2.31.0
810
github.com/HdrHistogram/hdrhistogram-go v1.1.2
911
github.com/Shopify/sarama v1.37.2
1012
github.com/apache/thrift v0.21.0
@@ -105,10 +107,22 @@ require (
105107
)
106108

107109
require (
110+
github.com/andybalholm/brotli v1.1.1 // indirect
111+
github.com/dmarkham/enumer v1.5.10 // indirect
112+
github.com/go-faster/city v1.0.1 // indirect
113+
github.com/go-faster/errors v0.7.1 // indirect
108114
github.com/gogo/googleapis v1.4.1 // indirect
115+
github.com/jackc/puddle/v2 v2.2.2 // indirect
109116
github.com/open-telemetry/opentelemetry-collector-contrib/pkg/core/xidutils v0.119.0 // indirect
117+
github.com/pascaldekloe/name v1.0.1 // indirect
118+
github.com/paulmach/orb v0.11.1 // indirect
119+
github.com/segmentio/asm v1.2.0 // indirect
120+
github.com/shopspring/decimal v1.4.0 // indirect
110121
go.opentelemetry.io/collector/extension/xextension v0.119.0 // indirect
111122
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.10.0 // indirect
123+
golang.org/x/mod v0.19.0 // indirect
124+
golang.org/x/sync v0.11.0 // indirect
125+
golang.org/x/tools v0.23.0 // indirect
112126
)
113127

114128
require (
@@ -264,7 +278,7 @@ require (
264278
go.opentelemetry.io/collector/processor/xprocessor v0.119.0 // indirect
265279
go.opentelemetry.io/collector/receiver/receivertest v0.119.0 // indirect
266280
go.opentelemetry.io/collector/receiver/xreceiver v0.119.0 // indirect
267-
go.opentelemetry.io/collector/semconv v0.119.0 // indirect
281+
go.opentelemetry.io/collector/semconv v0.119.0
268282
go.opentelemetry.io/collector/service v0.119.0 // indirect
269283
go.opentelemetry.io/contrib/bridges/otelzap v0.9.0 // indirect
270284
go.opentelemetry.io/contrib/config v0.14.0 // indirect

0 commit comments

Comments
 (0)