Skip to content

Commit 2b7f7df

Browse files
author
ligang
committed
Use time_series compaction for daily-partitioned trace/log/metric tables
1 parent fa94940 commit 2b7f7df

3 files changed

Lines changed: 96 additions & 32 deletions

File tree

‎deploy/common/sql/databuff.sql‎

Lines changed: 59 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,8 @@ PROPERTIES (
7676
"dynamic_partition.time_unit" = "DAY",
7777
"dynamic_partition.start" = "-30",
7878
"dynamic_partition.end" = "3",
79-
"dynamic_partition.prefix" = "p"
79+
"dynamic_partition.prefix" = "p",
80+
"compaction_policy" = "time_series"
8081
);
8182

8283
-- OTLP logs (v0.2.0+)
@@ -105,7 +106,8 @@ PROPERTIES (
105106
"dynamic_partition.time_unit" = "DAY",
106107
"dynamic_partition.start" = "-30",
107108
"dynamic_partition.end" = "3",
108-
"dynamic_partition.prefix" = "p"
109+
"dynamic_partition.prefix" = "p",
110+
"compaction_policy" = "time_series"
109111
);
110112

111113
-- metric (daily dynamic partition on metric_time; ts = epoch millis; default 30-day retention)
@@ -160,7 +162,8 @@ PROPERTIES (
160162
"dynamic_partition.time_unit" = "DAY",
161163
"dynamic_partition.start" = "-30",
162164
"dynamic_partition.end" = "3",
163-
"dynamic_partition.prefix" = "p"
165+
"dynamic_partition.prefix" = "p",
166+
"compaction_policy" = "time_series"
164167
);
165168

166169
CREATE TABLE metric_service (
@@ -198,7 +201,8 @@ PROPERTIES (
198201
"dynamic_partition.time_unit" = "DAY",
199202
"dynamic_partition.start" = "-30",
200203
"dynamic_partition.end" = "3",
201-
"dynamic_partition.prefix" = "p"
204+
"dynamic_partition.prefix" = "p",
205+
"compaction_policy" = "time_series"
202206
);
203207

204208
CREATE TABLE metric_service_config (
@@ -250,7 +254,8 @@ PROPERTIES (
250254
"dynamic_partition.time_unit" = "DAY",
251255
"dynamic_partition.start" = "-30",
252256
"dynamic_partition.end" = "3",
253-
"dynamic_partition.prefix" = "p"
257+
"dynamic_partition.prefix" = "p",
258+
"compaction_policy" = "time_series"
254259
);
255260

256261
CREATE TABLE metric_service_cpu (
@@ -275,7 +280,8 @@ PROPERTIES (
275280
"dynamic_partition.time_unit" = "DAY",
276281
"dynamic_partition.start" = "-30",
277282
"dynamic_partition.end" = "3",
278-
"dynamic_partition.prefix" = "p"
283+
"dynamic_partition.prefix" = "p",
284+
"compaction_policy" = "time_series"
279285
);
280286

281287
CREATE TABLE metric_service_db (
@@ -338,7 +344,8 @@ PROPERTIES (
338344
"dynamic_partition.time_unit" = "DAY",
339345
"dynamic_partition.start" = "-30",
340346
"dynamic_partition.end" = "3",
341-
"dynamic_partition.prefix" = "p"
347+
"dynamic_partition.prefix" = "p",
348+
"compaction_policy" = "time_series"
342349
);
343350

344351
CREATE TABLE metric_service_db_connection_pool (
@@ -376,7 +383,8 @@ PROPERTIES (
376383
"dynamic_partition.time_unit" = "DAY",
377384
"dynamic_partition.start" = "-30",
378385
"dynamic_partition.end" = "3",
379-
"dynamic_partition.prefix" = "p"
386+
"dynamic_partition.prefix" = "p",
387+
"compaction_policy" = "time_series"
380388
);
381389

382390
CREATE TABLE metric_service_db_connection_pool_get (
@@ -402,7 +410,8 @@ PROPERTIES (
402410
"dynamic_partition.time_unit" = "DAY",
403411
"dynamic_partition.start" = "-30",
404412
"dynamic_partition.end" = "3",
405-
"dynamic_partition.prefix" = "p"
413+
"dynamic_partition.prefix" = "p",
414+
"compaction_policy" = "time_series"
406415
);
407416

408417
CREATE TABLE metric_service_exception (
@@ -446,7 +455,8 @@ PROPERTIES (
446455
"dynamic_partition.time_unit" = "DAY",
447456
"dynamic_partition.start" = "-30",
448457
"dynamic_partition.end" = "3",
449-
"dynamic_partition.prefix" = "p"
458+
"dynamic_partition.prefix" = "p",
459+
"compaction_policy" = "time_series"
450460
);
451461

452462
CREATE TABLE metric_service_flow (
@@ -493,7 +503,8 @@ PROPERTIES (
493503
"dynamic_partition.time_unit" = "DAY",
494504
"dynamic_partition.start" = "-30",
495505
"dynamic_partition.end" = "3",
496-
"dynamic_partition.prefix" = "p"
506+
"dynamic_partition.prefix" = "p",
507+
"compaction_policy" = "time_series"
497508
);
498509

499510
CREATE TABLE metric_service_health_status (
@@ -530,7 +541,8 @@ PROPERTIES (
530541
"dynamic_partition.time_unit" = "DAY",
531542
"dynamic_partition.start" = "-30",
532543
"dynamic_partition.end" = "3",
533-
"dynamic_partition.prefix" = "p"
544+
"dynamic_partition.prefix" = "p",
545+
"compaction_policy" = "time_series"
534546
);
535547

536548
CREATE TABLE metric_service_http (
@@ -589,7 +601,8 @@ PROPERTIES (
589601
"dynamic_partition.time_unit" = "DAY",
590602
"dynamic_partition.start" = "-30",
591603
"dynamic_partition.end" = "3",
592-
"dynamic_partition.prefix" = "p"
604+
"dynamic_partition.prefix" = "p",
605+
"compaction_policy" = "time_series"
593606
);
594607

595608
CREATE TABLE metric_service_http_connection_pool (
@@ -617,7 +630,8 @@ PROPERTIES (
617630
"dynamic_partition.time_unit" = "DAY",
618631
"dynamic_partition.start" = "-30",
619632
"dynamic_partition.end" = "3",
620-
"dynamic_partition.prefix" = "p"
633+
"dynamic_partition.prefix" = "p",
634+
"compaction_policy" = "time_series"
621635
);
622636

623637
CREATE TABLE metric_service_http_connection_pool_get (
@@ -643,7 +657,8 @@ PROPERTIES (
643657
"dynamic_partition.time_unit" = "DAY",
644658
"dynamic_partition.start" = "-30",
645659
"dynamic_partition.end" = "3",
646-
"dynamic_partition.prefix" = "p"
660+
"dynamic_partition.prefix" = "p",
661+
"compaction_policy" = "time_series"
647662
);
648663

649664
CREATE TABLE metric_service_instance (
@@ -698,7 +713,8 @@ PROPERTIES (
698713
"dynamic_partition.time_unit" = "DAY",
699714
"dynamic_partition.start" = "-30",
700715
"dynamic_partition.end" = "3",
701-
"dynamic_partition.prefix" = "p"
716+
"dynamic_partition.prefix" = "p",
717+
"compaction_policy" = "time_series"
702718
);
703719

704720
CREATE TABLE metric_service_io (
@@ -724,7 +740,8 @@ PROPERTIES (
724740
"dynamic_partition.time_unit" = "DAY",
725741
"dynamic_partition.start" = "-30",
726742
"dynamic_partition.end" = "3",
727-
"dynamic_partition.prefix" = "p"
743+
"dynamic_partition.prefix" = "p",
744+
"compaction_policy" = "time_series"
728745
);
729746

730747
CREATE TABLE metric_service_mem (
@@ -751,7 +768,8 @@ PROPERTIES (
751768
"dynamic_partition.time_unit" = "DAY",
752769
"dynamic_partition.start" = "-30",
753770
"dynamic_partition.end" = "3",
754-
"dynamic_partition.prefix" = "p"
771+
"dynamic_partition.prefix" = "p",
772+
"compaction_policy" = "time_series"
755773
);
756774

757775
CREATE TABLE metric_service_mq (
@@ -814,7 +832,8 @@ PROPERTIES (
814832
"dynamic_partition.time_unit" = "DAY",
815833
"dynamic_partition.start" = "-30",
816834
"dynamic_partition.end" = "3",
817-
"dynamic_partition.prefix" = "p"
835+
"dynamic_partition.prefix" = "p",
836+
"compaction_policy" = "time_series"
818837
);
819838

820839
CREATE TABLE metric_service_net (
@@ -840,7 +859,8 @@ PROPERTIES (
840859
"dynamic_partition.time_unit" = "DAY",
841860
"dynamic_partition.start" = "-30",
842861
"dynamic_partition.end" = "3",
843-
"dynamic_partition.prefix" = "p"
862+
"dynamic_partition.prefix" = "p",
863+
"compaction_policy" = "time_series"
844864
);
845865

846866
CREATE TABLE metric_service_object_pool (
@@ -871,7 +891,8 @@ PROPERTIES (
871891
"dynamic_partition.time_unit" = "DAY",
872892
"dynamic_partition.start" = "-30",
873893
"dynamic_partition.end" = "3",
874-
"dynamic_partition.prefix" = "p"
894+
"dynamic_partition.prefix" = "p",
895+
"compaction_policy" = "time_series"
875896
);
876897

877898
CREATE TABLE metric_service_object_pool_get (
@@ -897,7 +918,8 @@ PROPERTIES (
897918
"dynamic_partition.time_unit" = "DAY",
898919
"dynamic_partition.start" = "-30",
899920
"dynamic_partition.end" = "3",
900-
"dynamic_partition.prefix" = "p"
921+
"dynamic_partition.prefix" = "p",
922+
"compaction_policy" = "time_series"
901923
);
902924

903925
CREATE TABLE metric_service_redis (
@@ -949,7 +971,8 @@ PROPERTIES (
949971
"dynamic_partition.time_unit" = "DAY",
950972
"dynamic_partition.start" = "-30",
951973
"dynamic_partition.end" = "3",
952-
"dynamic_partition.prefix" = "p"
974+
"dynamic_partition.prefix" = "p",
975+
"compaction_policy" = "time_series"
953976
);
954977

955978
CREATE TABLE metric_service_rpc (
@@ -1006,7 +1029,8 @@ PROPERTIES (
10061029
"dynamic_partition.time_unit" = "DAY",
10071030
"dynamic_partition.start" = "-30",
10081031
"dynamic_partition.end" = "3",
1009-
"dynamic_partition.prefix" = "p"
1032+
"dynamic_partition.prefix" = "p",
1033+
"compaction_policy" = "time_series"
10101034
);
10111035

10121036
CREATE TABLE metric_service_remote (
@@ -1061,7 +1085,8 @@ PROPERTIES (
10611085
"dynamic_partition.time_unit" = "DAY",
10621086
"dynamic_partition.start" = "-30",
10631087
"dynamic_partition.end" = "3",
1064-
"dynamic_partition.prefix" = "p"
1088+
"dynamic_partition.prefix" = "p",
1089+
"compaction_policy" = "time_series"
10651090
);
10661091

10671092
CREATE TABLE metric_service_tcp (
@@ -1087,7 +1112,8 @@ PROPERTIES (
10871112
"dynamic_partition.time_unit" = "DAY",
10881113
"dynamic_partition.start" = "-30",
10891114
"dynamic_partition.end" = "3",
1090-
"dynamic_partition.prefix" = "p"
1115+
"dynamic_partition.prefix" = "p",
1116+
"compaction_policy" = "time_series"
10911117
);
10921118

10931119
CREATE TABLE metric_service_thread_pool (
@@ -1120,7 +1146,8 @@ PROPERTIES (
11201146
"dynamic_partition.time_unit" = "DAY",
11211147
"dynamic_partition.start" = "-30",
11221148
"dynamic_partition.end" = "3",
1123-
"dynamic_partition.prefix" = "p"
1149+
"dynamic_partition.prefix" = "p",
1150+
"compaction_policy" = "time_series"
11241151
);
11251152

11261153
CREATE TABLE metric_service_thread_pool_cost (
@@ -1152,7 +1179,8 @@ PROPERTIES (
11521179
"dynamic_partition.time_unit" = "DAY",
11531180
"dynamic_partition.start" = "-30",
11541181
"dynamic_partition.end" = "3",
1155-
"dynamic_partition.prefix" = "p"
1182+
"dynamic_partition.prefix" = "p",
1183+
"compaction_policy" = "time_series"
11561184
);
11571185

11581186
CREATE TABLE metric_service_trace (
@@ -1191,7 +1219,8 @@ PROPERTIES (
11911219
"dynamic_partition.time_unit" = "DAY",
11921220
"dynamic_partition.start" = "-30",
11931221
"dynamic_partition.end" = "3",
1194-
"dynamic_partition.prefix" = "p"
1222+
"dynamic_partition.prefix" = "p",
1223+
"compaction_policy" = "time_series"
11951224
);
11961225

11971226
-- config
@@ -1596,4 +1625,4 @@ DISTRIBUTED BY HASH(`id`) BUCKETS 1
15961625
PROPERTIES ("replication_num" = "1");
15971626

15981627
INSERT INTO schema_version (id, version, applied_at)
1599-
VALUES (1, 5, NOW());
1628+
VALUES (1, 6, NOW());

‎deploy/common/sql/migrations/V006__time_series_compaction.sql‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
-- Set time_series compaction for append-only telemetry tables (trace / log / metric).
2+
-- Metadata-only: changes compaction_policy; no data rewrite. Defaults for
3+
-- time_series_compaction_* thresholds are left unchanged.
4+
5+
USE databuff;
6+
7+
ALTER TABLE trace_dc_span SET ("compaction_policy" = "time_series");
8+
ALTER TABLE log_dc_record SET ("compaction_policy" = "time_series");
9+
ALTER TABLE metric_jvm SET ("compaction_policy" = "time_series");
10+
ALTER TABLE metric_service SET ("compaction_policy" = "time_series");
11+
ALTER TABLE metric_service_config SET ("compaction_policy" = "time_series");
12+
ALTER TABLE metric_service_cpu SET ("compaction_policy" = "time_series");
13+
ALTER TABLE metric_service_db SET ("compaction_policy" = "time_series");
14+
ALTER TABLE metric_service_db_connection_pool SET ("compaction_policy" = "time_series");
15+
ALTER TABLE metric_service_db_connection_pool_get SET ("compaction_policy" = "time_series");
16+
ALTER TABLE metric_service_exception SET ("compaction_policy" = "time_series");
17+
ALTER TABLE metric_service_flow SET ("compaction_policy" = "time_series");
18+
ALTER TABLE metric_service_health_status SET ("compaction_policy" = "time_series");
19+
ALTER TABLE metric_service_http SET ("compaction_policy" = "time_series");
20+
ALTER TABLE metric_service_http_connection_pool SET ("compaction_policy" = "time_series");
21+
ALTER TABLE metric_service_http_connection_pool_get SET ("compaction_policy" = "time_series");
22+
ALTER TABLE metric_service_instance SET ("compaction_policy" = "time_series");
23+
ALTER TABLE metric_service_io SET ("compaction_policy" = "time_series");
24+
ALTER TABLE metric_service_mem SET ("compaction_policy" = "time_series");
25+
ALTER TABLE metric_service_mq SET ("compaction_policy" = "time_series");
26+
ALTER TABLE metric_service_net SET ("compaction_policy" = "time_series");
27+
ALTER TABLE metric_service_object_pool SET ("compaction_policy" = "time_series");
28+
ALTER TABLE metric_service_object_pool_get SET ("compaction_policy" = "time_series");
29+
ALTER TABLE metric_service_redis SET ("compaction_policy" = "time_series");
30+
ALTER TABLE metric_service_rpc SET ("compaction_policy" = "time_series");
31+
ALTER TABLE metric_service_remote SET ("compaction_policy" = "time_series");
32+
ALTER TABLE metric_service_tcp SET ("compaction_policy" = "time_series");
33+
ALTER TABLE metric_service_thread_pool SET ("compaction_policy" = "time_series");
34+
ALTER TABLE metric_service_thread_pool_cost SET ("compaction_policy" = "time_series");
35+
ALTER TABLE metric_service_trace SET ("compaction_policy" = "time_series");

‎deploy/common/upgrade-manifest.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
2-
"schema_version": 5,
2+
"schema_version": 6,
33
"upgrade_type": "A",
44
"min_upgrade_from": "0.1.0",
5-
"notes": "V005 reduces dynamic-partition telemetry table buckets from 16 to 3 for newly created partitions (existing partitions unchanged until retention)."
5+
"notes": "V006 sets compaction_policy=time_series on trace/log/metric dynamic-partition tables (append-only telemetry)."
66
}

0 commit comments

Comments
 (0)