Skip to content

Commit 8d39a87

Browse files
m-nagarajanm-nagarajan
committed
[da-vinci][server] Add OTel metrics to RocksDBMemoryStats
Add OTel ASYNC_GAUGE metrics for all 35 RocksDB memory sensors using the joint Tehuti+OTel API (AsyncMetricEntityStateBase.create). - Create RocksDBMemoryOtelMetricEntity enum (35 entries) as single source of truth for metric definitions and RocksDB property strings - Refactor RocksDBMemoryStats to use joint API, add inner TehutiMetricName enum, derive PARTITION_METRIC_DOMAINS and INSTANCE_METRIC_DOMAINS from OTel enum (eliminating duplication) - Add clusterName parameter for VENICE_CLUSTER_NAME dimension - Update VeniceServer and DaVinciBackend callers to pass clusterName - Register in ServerMetricEntity (count 87 -> 122) - Add comprehensive tests: OTel integration, Tehuti regression, metric entity validation, Tehuti enum validation, NPE prevention
1 parent b68623b commit 8d39a87

10 files changed

Lines changed: 910 additions & 86 deletions

File tree

‎clients/da-vinci-client/src/main/java/com/linkedin/davinci/DaVinciBackend.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -186,7 +186,8 @@ public DaVinciBackend(
186186
? new RocksDBMemoryStats(
187187
metricsRepository,
188188
"RocksDBMemoryStats",
189-
backendConfig.getRocksDBServerConfig().isRocksDBPlainTableFormatEnabled())
189+
backendConfig.getRocksDBServerConfig().isRocksDBPlainTableFormatEnabled(),
190+
configLoader.getVeniceClusterConfig().getClusterName())
190191
: null;
191192

192193
/**

‎clients/da-vinci-client/src/main/java/com/linkedin/davinci/stats/RocksDBMemoryOtelMetricEntity.java‎

Lines changed: 194 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,194 @@
1+
package com.linkedin.davinci.stats;
2+
3+
import static com.linkedin.venice.stats.dimensions.VeniceMetricsDimensions.VENICE_CLUSTER_NAME;
4+
import static com.linkedin.venice.utils.Utils.setOf;
5+
6+
import com.linkedin.venice.stats.metrics.MetricEntity;
7+
import com.linkedin.venice.stats.metrics.MetricType;
8+
import com.linkedin.venice.stats.metrics.MetricUnit;
9+
import com.linkedin.venice.stats.metrics.ModuleMetricEntityInterface;
10+
import java.util.Collections;
11+
import java.util.HashMap;
12+
import java.util.Map;
13+
14+
15+
/**
16+
* OTel metric entities for RocksDB memory consumption stats.
17+
*
18+
* <p>Maps 1:1 from Tehuti async gauges in {@link RocksDBMemoryStats} to OTel ASYNC_GAUGE metrics.
19+
*
20+
* <p>Constants that correspond to a RocksDB property (queried via
21+
* {@link com.linkedin.davinci.store.rocksdb.RocksDBStoragePartition#getRocksDBStatValue})
22+
* carry the original hyphenated property name (e.g. {@code "rocksdb.num-immutable-mem-table"})
23+
* and are discoverable via {@link #fromRocksDBProperty(String)}.
24+
*/
25+
public enum RocksDBMemoryOtelMetricEntity implements ModuleMetricEntityInterface {
26+
NUM_IMMUTABLE_MEM_TABLE(
27+
"rocksdb.num_immutable_mem_table", MetricUnit.NUMBER, "Number of immutable memtables",
28+
"rocksdb.num-immutable-mem-table"
29+
),
30+
MEM_TABLE_FLUSH_PENDING(
31+
"rocksdb.mem_table_flush_pending", MetricUnit.NUMBER, "Number of pending memtable flushes",
32+
"rocksdb.mem-table-flush-pending"
33+
),
34+
COMPACTION_PENDING(
35+
"rocksdb.compaction_pending", MetricUnit.NUMBER, "Number of pending compactions", "rocksdb.compaction-pending"
36+
),
37+
BACKGROUND_ERRORS(
38+
"rocksdb.background_errors", MetricUnit.NUMBER, "Number of background errors", "rocksdb.background-errors"
39+
),
40+
CUR_SIZE_ACTIVE_MEM_TABLE(
41+
"rocksdb.cur_size_active_mem_table", MetricUnit.BYTES, "Current size of active memtables",
42+
"rocksdb.cur-size-active-mem-table"
43+
),
44+
CUR_SIZE_ALL_MEM_TABLES(
45+
"rocksdb.cur_size_all_mem_tables", MetricUnit.BYTES, "Current size of all memtables",
46+
"rocksdb.cur-size-all-mem-tables"
47+
),
48+
SIZE_ALL_MEM_TABLES(
49+
"rocksdb.size_all_mem_tables", MetricUnit.BYTES, "Total size of all memtables", "rocksdb.size-all-mem-tables"
50+
),
51+
NUM_ENTRIES_ACTIVE_MEM_TABLE(
52+
"rocksdb.num_entries_active_mem_table", MetricUnit.NUMBER, "Number of entries in active memtables",
53+
"rocksdb.num-entries-active-mem-table"
54+
),
55+
NUM_ENTRIES_IMM_MEM_TABLES(
56+
"rocksdb.num_entries_imm_mem_tables", MetricUnit.NUMBER, "Number of entries in immutable memtables",
57+
"rocksdb.num-entries-imm-mem-tables"
58+
),
59+
NUM_DELETES_ACTIVE_MEM_TABLE(
60+
"rocksdb.num_deletes_active_mem_table", MetricUnit.NUMBER, "Number of deletes in active memtables",
61+
"rocksdb.num-deletes-active-mem-table"
62+
),
63+
NUM_DELETES_IMM_MEM_TABLES(
64+
"rocksdb.num_deletes_imm_mem_tables", MetricUnit.NUMBER, "Number of deletes in immutable memtables",
65+
"rocksdb.num-deletes-imm-mem-tables"
66+
),
67+
ESTIMATE_NUM_KEYS(
68+
"rocksdb.estimate_num_keys", MetricUnit.NUMBER, "Estimated number of keys", "rocksdb.estimate-num-keys"
69+
),
70+
ESTIMATE_TABLE_READERS_MEM(
71+
"rocksdb.estimate_table_readers_mem", MetricUnit.BYTES, "Estimated memory used by table readers",
72+
"rocksdb.estimate-table-readers-mem"
73+
), NUM_SNAPSHOTS("rocksdb.num_snapshots", MetricUnit.NUMBER, "Number of snapshots", "rocksdb.num-snapshots"),
74+
NUM_LIVE_VERSIONS(
75+
"rocksdb.num_live_versions", MetricUnit.NUMBER, "Number of live versions", "rocksdb.num-live-versions"
76+
),
77+
ESTIMATE_LIVE_DATA_SIZE(
78+
"rocksdb.estimate_live_data_size", MetricUnit.BYTES, "Estimated live data size", "rocksdb.estimate-live-data-size"
79+
),
80+
MIN_LOG_NUMBER_TO_KEEP(
81+
"rocksdb.min_log_number_to_keep", MetricUnit.NUMBER, "Minimum log number to keep",
82+
"rocksdb.min-log-number-to-keep"
83+
),
84+
TOTAL_SST_FILES_SIZE(
85+
"rocksdb.total_sst_files_size", MetricUnit.BYTES, "Total SST file size", "rocksdb.total-sst-files-size"
86+
),
87+
LIVE_SST_FILES_SIZE(
88+
"rocksdb.live_sst_files_size", MetricUnit.BYTES, "Live SST file size", "rocksdb.live-sst-files-size"
89+
),
90+
ESTIMATE_PENDING_COMPACTION_BYTES(
91+
"rocksdb.estimate_pending_compaction_bytes", MetricUnit.BYTES, "Estimated pending compaction bytes",
92+
"rocksdb.estimate-pending-compaction-bytes"
93+
),
94+
NUM_RUNNING_COMPACTIONS(
95+
"rocksdb.num_running_compactions", MetricUnit.NUMBER, "Number of running compactions",
96+
"rocksdb.num-running-compactions"
97+
),
98+
NUM_RUNNING_FLUSHES(
99+
"rocksdb.num_running_flushes", MetricUnit.NUMBER, "Number of running flushes", "rocksdb.num-running-flushes"
100+
),
101+
ACTUAL_DELAYED_WRITE_RATE(
102+
"rocksdb.actual_delayed_write_rate", MetricUnit.NUMBER, "Actual delayed write rate",
103+
"rocksdb.actual-delayed-write-rate"
104+
),
105+
106+
// Block cache properties
107+
BLOCK_CACHE_CAPACITY(
108+
"rocksdb.block_cache_capacity", MetricUnit.BYTES, "Block cache capacity", "rocksdb.block-cache-capacity"
109+
),
110+
BLOCK_CACHE_PINNED_USAGE(
111+
"rocksdb.block_cache_pinned_usage", MetricUnit.BYTES, "Block cache pinned usage",
112+
"rocksdb.block-cache-pinned-usage"
113+
), BLOCK_CACHE_USAGE("rocksdb.block_cache_usage", MetricUnit.BYTES, "Block cache usage", "rocksdb.block-cache-usage"),
114+
115+
// Blob file metrics
116+
NUM_BLOB_FILES("rocksdb.num_blob_files", MetricUnit.NUMBER, "Number of blob files", "rocksdb.num-blob-files"),
117+
TOTAL_BLOB_FILE_SIZE(
118+
"rocksdb.total_blob_file_size", MetricUnit.BYTES, "Total blob file size", "rocksdb.total-blob-file-size"
119+
),
120+
LIVE_BLOB_FILE_SIZE(
121+
"rocksdb.live_blob_file_size", MetricUnit.BYTES, "Live blob file size", "rocksdb.live-blob-file-size"
122+
),
123+
LIVE_BLOB_FILE_GARBAGE_SIZE(
124+
"rocksdb.live_blob_file_garbage_size", MetricUnit.BYTES, "Live blob file garbage size",
125+
"rocksdb.live-blob-file-garbage-size"
126+
),
127+
128+
// Server-level memory metrics (not backed by a RocksDB property)
129+
MEMORY_LIMIT("rocksdb.memory_limit", MetricUnit.BYTES, "RocksDB memory limit for this server"),
130+
MEMORY_USAGE("rocksdb.memory_usage", MetricUnit.BYTES, "RocksDB SST file manager total size"),
131+
132+
// RMD (Replication Metadata) block cache metrics (not backed by a RocksDB property)
133+
RMD_BLOCK_CACHE_CAPACITY("rocksdb.rmd_block_cache_capacity", MetricUnit.BYTES, "RMD block cache capacity"),
134+
RMD_BLOCK_CACHE_USAGE("rocksdb.rmd_block_cache_usage", MetricUnit.BYTES, "RMD block cache usage"),
135+
RMD_BLOCK_CACHE_PINNED_USAGE(
136+
"rocksdb.rmd_block_cache_pinned_usage", MetricUnit.BYTES, "RMD block cache pinned usage"
137+
);
138+
139+
/**
140+
* Lookup map from RocksDB property name (hyphenated, e.g. "rocksdb.num-immutable-mem-table")
141+
* to enum constant. Auto-built from constants that carry a non-null {@link #rocksDBProperty}.
142+
*/
143+
private static final Map<String, RocksDBMemoryOtelMetricEntity> PROPERTY_LOOKUP;
144+
145+
static {
146+
Map<String, RocksDBMemoryOtelMetricEntity> map = new HashMap<>();
147+
for (RocksDBMemoryOtelMetricEntity entity: values()) {
148+
if (entity.rocksDBProperty != null) {
149+
map.put(entity.rocksDBProperty, entity);
150+
}
151+
}
152+
PROPERTY_LOOKUP = Collections.unmodifiableMap(map);
153+
}
154+
155+
private final MetricEntity metricEntity;
156+
157+
/** Original RocksDB property name (hyphenated), or {@code null} for synthetic metrics. */
158+
private final String rocksDBProperty;
159+
160+
/** Returns the original RocksDB property name (hyphenated), or {@code null} for synthetic metrics. */
161+
public String getRocksDBProperty() {
162+
return rocksDBProperty;
163+
}
164+
165+
/** Constructor for metrics not backed by a RocksDB property (MEMORY_LIMIT, MEMORY_USAGE, RMD_*). */
166+
RocksDBMemoryOtelMetricEntity(String metricName, MetricUnit unit, String description) {
167+
this(metricName, unit, description, null);
168+
}
169+
170+
/** Constructor for metrics backed by a RocksDB property. */
171+
RocksDBMemoryOtelMetricEntity(String metricName, MetricUnit unit, String description, String rocksDBProperty) {
172+
this.rocksDBProperty = rocksDBProperty;
173+
this.metricEntity =
174+
new MetricEntity(metricName, MetricType.ASYNC_GAUGE, unit, description, setOf(VENICE_CLUSTER_NAME));
175+
}
176+
177+
@Override
178+
public MetricEntity getMetricEntity() {
179+
return metricEntity;
180+
}
181+
182+
/**
183+
* Returns the OTel metric entity for a given RocksDB property name (e.g., "rocksdb.num-immutable-mem-table").
184+
*
185+
* @throws IllegalArgumentException if the property name is not recognized
186+
*/
187+
public static RocksDBMemoryOtelMetricEntity fromRocksDBProperty(String propertyName) {
188+
RocksDBMemoryOtelMetricEntity entity = PROPERTY_LOOKUP.get(propertyName);
189+
if (entity == null) {
190+
throw new IllegalArgumentException("Unknown RocksDB property: " + propertyName);
191+
}
192+
return entity;
193+
}
194+
}

0 commit comments

Comments
 (0)