diff --git a/docs/operations/metrics.md b/docs/operations/metrics.md index 6f87b7f5c963..b77689f77628 100644 --- a/docs/operations/metrics.md +++ b/docs/operations/metrics.md @@ -428,14 +428,14 @@ These metrics are emitted by the Druid Coordinator in every run of the correspon |`segment/loadQueue/count`|Number of segments to load.|`server`|Varies| |`segment/loading/rateKbps`|Current rate of segment loading on a server in kbps (1000 bits per second). The rate is calculated as a moving average over the last 10 GiB or more of successful segment loads on that server.|`server`|Varies| |`segment/dropQueue/count`|Number of segments to drop.|`server`|Varies| -|`segment/loadQueue/assigned`|Number of segments assigned for load or drop to the load queue of a server.|`dataSource`, `server`|Varies| -|`segment/loadQueue/success`|Number of segment assignments that completed successfully.|`dataSource`, `server`|Varies| -|`segment/loadQueue/failed`|Number of segment assignments that failed to complete.|`dataSource`, `server`|0| -|`segment/loadQueue/cancelled`|Number of segment assignments that were canceled before completion.|`dataSource`, `server`|Varies| +|`segment/loadQueue/assigned`|Number of segments assigned for load or drop to the load queue of a server.|`dataSource`, `description`|Varies| +|`segment/loadQueue/success`|Number of segment assignments that completed successfully.|`dataSource`, `description`|Varies| +|`segment/loadQueue/failed`|Number of segment assignments that failed to complete.|`dataSource`, `description`|0| +|`segment/loadQueue/cancelled`|Number of segment assignments that were canceled before completion.|`dataSource`, `description`|Varies| |`segment/size`|Total size of used segments in a data source. Emitted only for data sources to which at least one used segment belongs.|`dataSource`|Varies| |`segment/count`|Number of used segments belonging to a data source. Emitted only for data sources to which at least one used segment belongs.|`dataSource`|< max| |`segment/overShadowed/count`|Number of segments marked as unused due to being overshadowed.| |Varies| -|`segment/unneededEternityTombstone/count`|Number of non-overshadowed eternity tombstones marked as unused.| |Varies| +|`segment/unneededEternityTombstone/count`|Number of non-overshadowed eternity tombstones marked as unused.|`dataSource`|Varies| |`segment/unavailable/count`|Number of unique segments left to load until all used segments are available for queries.|`dataSource`|0| |`segment/underReplicated/count`|Number of segments, including replicas, left to load until all used segments are available for queries.|`tier`, `dataSource`|0| |`segment/availableDeepStorageOnly/count`|Number of unique segments that are only available for querying directly from deep storage.|`dataSource`|Varies| diff --git a/extensions-contrib/statsd-emitter/src/main/resources/defaultMetricDimensions.json b/extensions-contrib/statsd-emitter/src/main/resources/defaultMetricDimensions.json index 828031cfb114..b5ebaf97530c 100644 --- a/extensions-contrib/statsd-emitter/src/main/resources/defaultMetricDimensions.json +++ b/extensions-contrib/statsd-emitter/src/main/resources/defaultMetricDimensions.json @@ -101,19 +101,33 @@ "segment/moved/bytes" : { "dimensions" : ["dataSource", "taskType"], "type" : "count" }, "segment/nuked/bytes" : { "dimensions" : ["dataSource", "taskType"], "type" : "count" }, - "segment/assigned/count" : { "dimensions" : ["tier"], "type" : "count" }, - "segment/moved/count" : { "dimensions" : ["tier"], "type" : "count" }, - "segment/dropped/count" : { "dimensions" : ["tier"], "type" : "count" }, - "segment/deleted/count" : { "dimensions" : ["tier"], "type" : "count" }, - "segment/unneeded/count" : { "dimensions" : ["tier"], "type" : "count" }, + "segment/assigned/count" : { "dimensions" : ["dataSource", "tier"], "type" : "count" }, + "segment/moved/count" : { "dimensions" : ["dataSource", "tier"], "type" : "count" }, + "segment/dropped/count" : { "dimensions" : ["dataSource", "tier"], "type" : "count" }, + "segment/deleted/count" : { "dimensions" : ["dataSource"], "type" : "count" }, + "segment/unneeded/count" : { "dimensions" : ["dataSource", "tier"], "type" : "count" }, "segment/unavailable/count" : { "dimensions" : ["dataSource"], "type" : "gauge" }, "segment/underReplicated/count" : { "dimensions" : ["dataSource", "tier"], "type" : "gauge" }, "segment/loadQueue/size" : { "dimensions" : ["server"], "type" : "gauge" }, - "segment/loadQueue/failed" : { "dimensions" : ["server"], "type" : "gauge" }, "segment/loadQueue/count" : { "dimensions" : ["server"], "type" : "gauge" }, "segment/dropQueue/count" : { "dimensions" : ["server"], "type" : "gauge" }, "segment/size" : { "dimensions" : ["dataSource"], "type" : "gauge" }, "segment/overShadowed/count" : { "dimensions" : [], "type" : "gauge" }, + "segment/assignSkipped/count" : { "dimensions" : ["dataSource", "server", "tier", "description"], "type" : "count" }, + "segment/dropSkipped/count" : { "dimensions" : ["dataSource", "server", "tier", "description"], "type" : "count" }, + "segment/moveSkipped/count" : { "dimensions" : ["dataSource", "server", "tier", "description"], "type" : "count" }, + + "segment/count" : { "dimensions" : ["dataSource"], "type" : "gauge" }, + "segment/availableDeepStorageOnly/count" : { "dimensions" : ["dataSource"], "type" : "gauge" }, + "segment/unneededEternityTombstone/count" : { "dimensions" : ["dataSource"], "type" : "gauge" }, + "segment/clone/assigned/count" : { "dimensions" : ["dataSource"], "type" : "count" }, + "segment/clone/dropped/count" : { "dimensions" : ["dataSource"], "type" : "count" }, + + "segment/loading/rateKbps" : { "dimensions" : ["server"], "type" : "gauge" }, + "segment/loadQueue/assigned" : { "dimensions" : ["dataSource", "description"], "type" : "gauge" }, + "segment/loadQueue/success" : { "dimensions" : ["dataSource", "description"], "type" : "gauge" }, + "segment/loadQueue/failed" : { "dimensions" : ["dataSource", "description"], "type" : "gauge" }, + "segment/loadQueue/cancelled" : { "dimensions" : ["dataSource", "description"], "type" : "gauge" }, "segment/metadataCache/sync/time": { "dimensions": [ "host", "service" ], "type" : "timer" }, @@ -178,10 +192,18 @@ "tier/replication/factor" : { "dimensions" : ["tier"], "type" : "gauge" }, "tier/historical/count" : { "dimensions" : ["tier"], "type" : "count" }, + "tier/storage/capacity" : { "dimensions" : ["tier"], "type" : "gauge" }, + "tier/historical/clone/count" : { "dimensions" : ["tier"], "type" : "count" }, + "compact/task/count" : { "dimensions" : [], "type" : "count" }, "compactTask/maxSlot/count" : { "dimensions" : [], "type" : "count" }, "compactTask/availableSlot/count" : { "dimensions" : [], "type" : "count" }, + "compact/createJobs/time" : { "dimensions" : [], "type" : "timer" }, + "compact/createJobs/count" : { "dimensions" : [], "type" : "count" }, + "compact/runScheduler/time" : { "dimensions" : [], "type" : "timer" }, + "compactTask/cancelled/count" : { "dimensions" : [], "type" : "count" }, + "segment/waitCompact/bytes" : { "dimensions" : ["dataSource"], "type" : "gauge" }, "segment/waitCompact/count" : { "dimensions" : ["dataSource"], "type" : "count" }, "interval/waitCompact/count" : { "dimensions" : ["dataSource"], "type" : "count" }, @@ -199,7 +221,22 @@ "service/heartbeat" : { "dimensions" : ["leader", "workerVersion", "category", "status", "taskId", "groupId", "dataSource", "taskStatus" ], "type" : "count" }, + "metadata/kill/compaction/count" : { "dimensions" : [], "type" : "count" }, + "metadata/kill/supervisor/count" : { "dimensions" : [], "type" : "count" }, + "metadata/kill/rule/count" : { "dimensions" : [], "type" : "count" }, + "metadata/kill/segmentSchema/count" : { "dimensions" : [], "type" : "count" }, + "metadata/kill/audit/count" : { "dimensions" : [], "type" : "count" }, + "metadata/kill/datasource/count" : { "dimensions" : [], "type" : "count" }, + "kill/eligibleUnusedSegments/count" : { "dimensions" : ["dataSource"], "type" : "gauge" }, + "kill/pendingSegments/count" : { "dimensions" : ["dataSource"], "type" : "count" }, + "killTask/availableSlot/count" : { "dimensions" : [], "type" : "count" }, "killTask/maxSlot/count" : { "dimensions" : [], "type" : "count" }, - "killTask/task/count" : { "dimensions" : [], "type" : "count" } + "kill/task/count" : { "dimensions" : [], "type" : "count" }, + + "segment/balancer/compute/error" : { "dimensions" : ["tier", "dataSource", "description"], "type" : "count" }, + + "config/brokerSync/time" : { "dimensions" : [], "type" : "timer" }, + "config/brokerSync/total/time" : { "dimensions" : [], "type" : "timer" }, + "config/brokerSync/error" : { "dimensions" : [], "type" : "count" } } diff --git a/server/src/main/java/org/apache/druid/server/coordinator/stats/Stats.java b/server/src/main/java/org/apache/druid/server/coordinator/stats/Stats.java index 2bf3772e287e..acc4f313f5b0 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/stats/Stats.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/stats/Stats.java @@ -58,7 +58,7 @@ public static class Segments public static final CoordinatorStat UNNEEDED = CoordinatorStat.toDebugAndEmit("unneeded", "segment/unneeded/count"); public static final CoordinatorStat OVERSHADOWED - = CoordinatorStat.toDebugAndEmit("overshadowed", "segment/overshadowed/count"); + = CoordinatorStat.toDebugAndEmit("overshadowed", "segment/overShadowed/count"); public static final CoordinatorStat UNNEEDED_ETERNITY_TOMBSTONE = CoordinatorStat.toDebugAndEmit("unneededEternityTombstone", "segment/unneededEternityTombstone/count"); diff --git a/server/src/test/java/org/apache/druid/server/coordinator/simulate/CoordinatorSimulationBaseTest.java b/server/src/test/java/org/apache/druid/server/coordinator/simulate/CoordinatorSimulationBaseTest.java index bcce6bcee060..42612b47e3d0 100644 --- a/server/src/test/java/org/apache/druid/server/coordinator/simulate/CoordinatorSimulationBaseTest.java +++ b/server/src/test/java/org/apache/druid/server/coordinator/simulate/CoordinatorSimulationBaseTest.java @@ -232,7 +232,7 @@ static class Metric static final String SUCCESS_ACTIONS = "segment/loadQueue/success"; static final String CANCELLED_ACTIONS = "segment/loadQueue/cancelled"; - static final String OVERSHADOWED_COUNT = "segment/overshadowed/count"; + static final String OVERSHADOWED_COUNT = "segment/overShadowed/count"; } static class Segments