From afa18e3e6b0a16377cb5e90d55a2c372dc1b069c Mon Sep 17 00:00:00 2001 From: Joe Adams Date: Sun, 21 Apr 2024 16:49:50 -0400 Subject: [PATCH 01/12] Start work on mixin dashboards Signed-off-by: Joe Adams --- .gitignore | 1 + elasticsearch_mixin/README.md | 20 ++++++++ elasticsearch_mixin/config.libsonnet | 6 +++ elasticsearch_mixin/dashboards.jsonnet | 3 ++ .../dashboards/cluster.libsonnet | 20 ++++++++ .../dashboards/dashboards.libsonnet | 1 + elasticsearch_mixin/dashboards/g.libsonnet | 1 + .../dashboards/variables.libsonnet | 15 ++++++ elasticsearch_mixin/jsonnetfile.json | 15 ++++++ elasticsearch_mixin/jsonnetfile.lock.json | 46 +++++++++++++++++++ elasticsearch_mixin/mixin.libsonnet | 3 ++ 11 files changed, 131 insertions(+) create mode 100644 elasticsearch_mixin/README.md create mode 100644 elasticsearch_mixin/config.libsonnet create mode 100644 elasticsearch_mixin/dashboards.jsonnet create mode 100644 elasticsearch_mixin/dashboards/cluster.libsonnet create mode 100644 elasticsearch_mixin/dashboards/dashboards.libsonnet create mode 100644 elasticsearch_mixin/dashboards/g.libsonnet create mode 100644 elasticsearch_mixin/dashboards/variables.libsonnet create mode 100644 elasticsearch_mixin/jsonnetfile.json create mode 100644 elasticsearch_mixin/jsonnetfile.lock.json create mode 100644 elasticsearch_mixin/mixin.libsonnet diff --git a/.gitignore b/.gitignore index e470da31..9687f990 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ elasticsearch_exporter *-stamp .tarballs /vendor +vendor/ diff --git a/elasticsearch_mixin/README.md b/elasticsearch_mixin/README.md new file mode 100644 index 00000000..dec45492 --- /dev/null +++ b/elasticsearch_mixin/README.md @@ -0,0 +1,20 @@ +# Elasticsearch Exporter Mixin + +This is a mixin for the elasticsearch_exporter to define dashboards, alerts, and monitoring queries for use with this exporter. + +Good example of upstream mixin for reference: https://github.com/kubernetes-monitoring/kubernetes-mixin + + +## Development + +### JSONNET +```go install github.com/google/go-jsonnet/cmd/jsonnet@latest``` + +### JSONNET BUNDLER +jsonnet bundler is a package manager for jsonnet + +```go install -a github.com/jsonnet-bundler/jsonnet-bundler/cmd/jb@latest``` + + +### Grafonnet +```jb install github.com/grafana/grafonnet/gen/grafonnet-latest@main``` diff --git a/elasticsearch_mixin/config.libsonnet b/elasticsearch_mixin/config.libsonnet new file mode 100644 index 00000000..3cf4a4f9 --- /dev/null +++ b/elasticsearch_mixin/config.libsonnet @@ -0,0 +1,6 @@ +{ + _config+:: { + dashboardNamePrefix: 'Elasticsearch Exporter / ', + dashboardTags: ['elasticsearch-exporter-mixin'], + }, +} diff --git a/elasticsearch_mixin/dashboards.jsonnet b/elasticsearch_mixin/dashboards.jsonnet new file mode 100644 index 00000000..d606bd6c --- /dev/null +++ b/elasticsearch_mixin/dashboards.jsonnet @@ -0,0 +1,3 @@ +local dashboards = (import 'mixin.libsonnet').grafanaDashboards; + +{ [name]: dashboards[name] for name in std.objectFields(dashboards)} diff --git a/elasticsearch_mixin/dashboards/cluster.libsonnet b/elasticsearch_mixin/dashboards/cluster.libsonnet new file mode 100644 index 00000000..abf8b9e5 --- /dev/null +++ b/elasticsearch_mixin/dashboards/cluster.libsonnet @@ -0,0 +1,20 @@ +local g = import 'g.libsonnet'; + +local dashboard = g.dashboard; + +local variables = import './variables.libsonnet'; + +{ + grafanaDashboards+:: { + 'cluster.json': + dashboard.new('%s Cluster' % $._config.dashboardNamePrefix) + + dashboard.withTags($._config.dashboardTags) + + dashboard.withRefresh('1m') + + dashboard.time.withFrom(value='now-1h') + + dashboard.graphTooltip.withSharedCrosshair() + + dashboard.withVariables([ + variables.datasource, + variables.cluster, + ]) + } +} diff --git a/elasticsearch_mixin/dashboards/dashboards.libsonnet b/elasticsearch_mixin/dashboards/dashboards.libsonnet new file mode 100644 index 00000000..16802c25 --- /dev/null +++ b/elasticsearch_mixin/dashboards/dashboards.libsonnet @@ -0,0 +1 @@ +(import 'cluster.libsonnet') diff --git a/elasticsearch_mixin/dashboards/g.libsonnet b/elasticsearch_mixin/dashboards/g.libsonnet new file mode 100644 index 00000000..71fdb755 --- /dev/null +++ b/elasticsearch_mixin/dashboards/g.libsonnet @@ -0,0 +1 @@ +import "github.com/grafana/grafonnet/gen/grafonnet-latest/main.libsonnet" diff --git a/elasticsearch_mixin/dashboards/variables.libsonnet b/elasticsearch_mixin/dashboards/variables.libsonnet new file mode 100644 index 00000000..c8d5715a --- /dev/null +++ b/elasticsearch_mixin/dashboards/variables.libsonnet @@ -0,0 +1,15 @@ +local g = import './g.libsonnet'; +local var = g.dashboard.variable; + +{ + datasource: + var.datasource.new('datasource', 'prometheus'), + + cluster: + var.query.new('cluster') + + var.query.withDatasourceFromVariable(self.datasource) + + var.query.queryTypes.withLabelValues( + 'cluster', + 'elasticsearch_cluster_health_status', + ) +} diff --git a/elasticsearch_mixin/jsonnetfile.json b/elasticsearch_mixin/jsonnetfile.json new file mode 100644 index 00000000..2414c867 --- /dev/null +++ b/elasticsearch_mixin/jsonnetfile.json @@ -0,0 +1,15 @@ +{ + "version": 1, + "dependencies": [ + { + "source": { + "git": { + "remote": "https://github.com/grafana/grafonnet.git", + "subdir": "gen/grafonnet-latest" + } + }, + "version": "main" + } + ], + "legacyImports": true +} diff --git a/elasticsearch_mixin/jsonnetfile.lock.json b/elasticsearch_mixin/jsonnetfile.lock.json new file mode 100644 index 00000000..31b59c22 --- /dev/null +++ b/elasticsearch_mixin/jsonnetfile.lock.json @@ -0,0 +1,46 @@ +{ + "version": 1, + "dependencies": [ + { + "source": { + "git": { + "remote": "https://github.com/grafana/grafonnet.git", + "subdir": "gen/grafonnet-latest" + } + }, + "version": "1c56af39815c4903e47c27194444456f005f65df", + "sum": "GxEO83uxgsDclLp/fmlUJZDbSGpeUZY6Ap3G2cgdL1g=" + }, + { + "source": { + "git": { + "remote": "https://github.com/grafana/grafonnet.git", + "subdir": "gen/grafonnet-v10.4.0" + } + }, + "version": "1c56af39815c4903e47c27194444456f005f65df", + "sum": "DKj+Sn+rlI48g/aoJpzkfPge46ya0jLk5kcZoiZ2X/I=" + }, + { + "source": { + "git": { + "remote": "https://github.com/jsonnet-libs/docsonnet.git", + "subdir": "doc-util" + } + }, + "version": "6ac6c69685b8c29c54515448eaca583da2d88150", + "sum": "BrAL/k23jq+xy9oA7TWIhUx07dsA/QLm3g7ktCwe//U=" + }, + { + "source": { + "git": { + "remote": "https://github.com/jsonnet-libs/xtd.git", + "subdir": "" + } + }, + "version": "63d430b69a95741061c2f7fc9d84b1a778511d9c", + "sum": "qiZi3axUSXCVzKUF83zSAxklwrnitMmrDK4XAfjPMdE=" + } + ], + "legacyImports": false +} diff --git a/elasticsearch_mixin/mixin.libsonnet b/elasticsearch_mixin/mixin.libsonnet new file mode 100644 index 00000000..7083cb16 --- /dev/null +++ b/elasticsearch_mixin/mixin.libsonnet @@ -0,0 +1,3 @@ +// (import 'alerts/alerts.libsonnet') + +(import 'dashboards/dashboards.libsonnet') + +(import 'config.libsonnet') From b625110255adf033e53564a21eda30c5871fea48 Mon Sep 17 00:00:00 2001 From: Joe Adams Date: Sun, 7 Jul 2024 15:27:11 -0400 Subject: [PATCH 02/12] Add overview and shards These rows are pretty close to complete. This starts the work on documents while the sizing is still off. Signed-off-by: Joe Adams --- .../dashboards/cluster.libsonnet | 33 ++++- .../dashboards/panels.libsonnet | 29 +++++ .../dashboards/queries.libsonnet | 115 ++++++++++++++++++ 3 files changed, 174 insertions(+), 3 deletions(-) create mode 100644 elasticsearch_mixin/dashboards/panels.libsonnet create mode 100644 elasticsearch_mixin/dashboards/queries.libsonnet diff --git a/elasticsearch_mixin/dashboards/cluster.libsonnet b/elasticsearch_mixin/dashboards/cluster.libsonnet index abf8b9e5..314a6e53 100644 --- a/elasticsearch_mixin/dashboards/cluster.libsonnet +++ b/elasticsearch_mixin/dashboards/cluster.libsonnet @@ -1,7 +1,10 @@ local g = import 'g.libsonnet'; local dashboard = g.dashboard; +local row = g.panel.row; +local panels = import './panels.libsonnet'; +local queries = import './queries.libsonnet'; local variables = import './variables.libsonnet'; { @@ -13,8 +16,32 @@ local variables = import './variables.libsonnet'; + dashboard.time.withFrom(value='now-1h') + dashboard.graphTooltip.withSharedCrosshair() + dashboard.withVariables([ - variables.datasource, - variables.cluster, + variables.datasource, + variables.cluster, ]) - } + + dashboard.withPanels( + g.util.grid.makeGrid([ + row.new('Overview') + + row.withPanels([ + panels.stat.nodes('Nodes', queries.runningNodes), + panels.stat.nodes('Data Nodes', queries.dataNodes), + panels.stat.nodes('Pending Tasks', queries.pendingTasks), + ]), + row.new('Shards') + + row.withPanels([ + panels.stat.nodes('Active', queries.activeShards), + panels.stat.nodes('Active Primary', queries.activePrimaryShards), + panels.stat.nodes('Initializing', queries.initializingShards), + panels.stat.nodes('Relocating', queries.reloactingShards), + panels.stat.nodes('Unassigned', queries.unassignedShards), + panels.stat.nodes('DelayedUnassigned', queries.delayedUnassignedShards), + ]), + row.new('Documents') + + row.withPanels([ + panels.timeSeries.base('Indexed Documents', queries.indexedDocuments), + + ]), + ], panelWidth=4, panelHeight=3), + ), + }, } diff --git a/elasticsearch_mixin/dashboards/panels.libsonnet b/elasticsearch_mixin/dashboards/panels.libsonnet new file mode 100644 index 00000000..38f92232 --- /dev/null +++ b/elasticsearch_mixin/dashboards/panels.libsonnet @@ -0,0 +1,29 @@ +local g = import 'g.libsonnet'; + +{ + stat: { + local stat = g.panel.stat, + + base(title, targets): + stat.new(title) + + stat.queryOptions.withTargets(targets), + + nodes: self.base, + }, + + timeSeries: { + local timeSeries = g.panel.timeSeries, + + base(title, targets): + timeSeries.new(title) + + timeSeries.queryOptions.withTargets(targets), + + ratio(title, targets): + self.base(title, targets) + + timeSeries.standardOptions.withUnit('percentunit'), + + ratioMax1(title, targets): + self.ratio(title, targets) + + timeSeries.standardOptions.withMax(1), + }, +} diff --git a/elasticsearch_mixin/dashboards/queries.libsonnet b/elasticsearch_mixin/dashboards/queries.libsonnet new file mode 100644 index 00000000..b9ced5d1 --- /dev/null +++ b/elasticsearch_mixin/dashboards/queries.libsonnet @@ -0,0 +1,115 @@ +local g = import './g.libsonnet'; +local prometheusQuery = g.query.prometheus; + +local variables = import './variables.libsonnet'; + +{ + /* + General overview queries + */ + + runningNodes: + prometheusQuery.new( + '$' + variables.datasource.name, + ||| + sum( + elasticsearch_cluster_health_number_of_nodes{cluster=~"$cluster"} + ) + ||| + ), + dataNodes: + prometheusQuery.new( + '$' + variables.datasource.name, + ||| + sum( + elasticsearch_cluster_health_number_of_data_nodes{cluster=~"$cluster"} + ) + ||| + ), + + pendingTasks: + prometheusQuery.new( + '$' + variables.datasource.name, + ||| + sum( + elasticsearch_cluster_health_number_of_pending_tasks{cluster=~"$cluster"} + ) + ||| + ), + + /* + Shard queries + */ + + activeShards: + prometheusQuery.new( + '$' + variables.datasource.name, + ||| + sum( + elasticsearch_cluster_health_active_shards{cluster=~"$cluster"} + ) + ||| + ), + + activePrimaryShards: + prometheusQuery.new( + '$' + variables.datasource.name, + ||| + sum( + elasticsearch_cluster_health_active_primary_shards{cluster=~"$cluster"} + ) + ||| + ), + + initializingShards: + prometheusQuery.new( + '$' + variables.datasource.name, + ||| + sum( + elasticsearch_cluster_health_initializing_shards{cluster=~"$cluster"} + ) + ||| + ), + + reloactingShards: + prometheusQuery.new( + '$' + variables.datasource.name, + ||| + sum( + elasticsearch_cluster_health_reloacting_shards{cluster=~"$cluster"} + ) + ||| + ), + + unassignedShards: + prometheusQuery.new( + '$' + variables.datasource.name, + ||| + sum( + elasticsearch_cluster_health_unassigned_shards{cluster=~"$cluster"} + ) + ||| + ), + + delayedUnassignedShards: + prometheusQuery.new( + '$' + variables.datasource.name, + ||| + sum( + elasticsearch_cluster_health_delayed_unassigned_shards{cluster=~"$cluster"} + ) + ||| + ), + + /* + Documents queries + */ + + indexedDocuments: + prometheusQuery.new( + '$' + variables.datasource.name, + ||| + elasticsearch_indices_docs{cluster=~"$cluster"} + ||| + ), +} From c8bfd2318ba10edba87e3fc560f8869d0d706a22 Mon Sep 17 00:00:00 2001 From: Joe Adams Date: Sun, 14 Jul 2024 01:16:22 -0400 Subject: [PATCH 03/12] Add remaining queries and panels from old dashboard Signed-off-by: Joe Adams --- .../dashboards/cluster.libsonnet | 21 +++- .../dashboards/panels.libsonnet | 11 +- .../dashboards/queries.libsonnet | 116 +----------------- .../dashboards/queries/document.libsonnet | 50 ++++++++ .../dashboards/queries/general.libsonnet | 35 ++++++ .../dashboards/queries/memory.libsonnet | 47 +++++++ .../dashboards/queries/network.libsonnet | 28 +++++ .../dashboards/queries/shard.libsonnet | 66 ++++++++++ .../dashboards/queries/threads.libsonnet | 24 ++++ 9 files changed, 286 insertions(+), 112 deletions(-) create mode 100644 elasticsearch_mixin/dashboards/queries/document.libsonnet create mode 100644 elasticsearch_mixin/dashboards/queries/general.libsonnet create mode 100644 elasticsearch_mixin/dashboards/queries/memory.libsonnet create mode 100644 elasticsearch_mixin/dashboards/queries/network.libsonnet create mode 100644 elasticsearch_mixin/dashboards/queries/shard.libsonnet create mode 100644 elasticsearch_mixin/dashboards/queries/threads.libsonnet diff --git a/elasticsearch_mixin/dashboards/cluster.libsonnet b/elasticsearch_mixin/dashboards/cluster.libsonnet index 314a6e53..45744279 100644 --- a/elasticsearch_mixin/dashboards/cluster.libsonnet +++ b/elasticsearch_mixin/dashboards/cluster.libsonnet @@ -39,7 +39,26 @@ local variables = import './variables.libsonnet'; row.new('Documents') + row.withPanels([ panels.timeSeries.base('Indexed Documents', queries.indexedDocuments), - + panels.timeSeries.bytes('Index Size', queries.indexSize), + panels.timeSeries.base('Index Rate', queries.indexRate), + panels.timeSeries.base('Query Rate', queries.queryRate), + panels.timeSeries.base('Queue Count', queries.queueCount), + ]), + row.new('Memory') + + row.withPanels([ + panels.timeSeries.bytes('Memory Usage', queries.memoryUsage), + panels.timeSeries.ratioMax1('Memory 15m Avg', queries.memoryUsageAverage15), + panels.timeSeries.bytes('Memory Max', queries.memoryMax), + panels.timeSeries.seconds('GC Rate', queries.gcSeconds), + ]), + row.new('Threads') + + row.withPanels([ + panels.timeSeries.base('Thread Pools', queries.threadPoolActive), + panels.timeSeries.base('Thread Pool Rejections', queries.threadPoolRejections), + ]), + row.new('Network') + + row.withPanels([ + panels.timeSeries.bytes('Transport Rate', [queries.transportTXRate, queries.transportRXRate]), ]), ], panelWidth=4, panelHeight=3), ), diff --git a/elasticsearch_mixin/dashboards/panels.libsonnet b/elasticsearch_mixin/dashboards/panels.libsonnet index 38f92232..f5f24d45 100644 --- a/elasticsearch_mixin/dashboards/panels.libsonnet +++ b/elasticsearch_mixin/dashboards/panels.libsonnet @@ -24,6 +24,15 @@ local g = import 'g.libsonnet'; ratioMax1(title, targets): self.ratio(title, targets) - + timeSeries.standardOptions.withMax(1), + + timeSeries.standardOptions.withMax(1) + + timeSeries.standardOptions.withMin(0), + + bytes(title, targets): + self.base(title, targets) + + timeSeries.standardOptions.withUnit('bytes'), + + seconds(title, targets): + self.base(title, targets) + + timeSeries.standardOptions.withUnit('s'), }, } diff --git a/elasticsearch_mixin/dashboards/queries.libsonnet b/elasticsearch_mixin/dashboards/queries.libsonnet index b9ced5d1..d7a8e661 100644 --- a/elasticsearch_mixin/dashboards/queries.libsonnet +++ b/elasticsearch_mixin/dashboards/queries.libsonnet @@ -3,113 +3,9 @@ local prometheusQuery = g.query.prometheus; local variables = import './variables.libsonnet'; -{ - /* - General overview queries - */ - - runningNodes: - prometheusQuery.new( - '$' + variables.datasource.name, - ||| - sum( - elasticsearch_cluster_health_number_of_nodes{cluster=~"$cluster"} - ) - ||| - ), - dataNodes: - prometheusQuery.new( - '$' + variables.datasource.name, - ||| - sum( - elasticsearch_cluster_health_number_of_data_nodes{cluster=~"$cluster"} - ) - ||| - ), - - pendingTasks: - prometheusQuery.new( - '$' + variables.datasource.name, - ||| - sum( - elasticsearch_cluster_health_number_of_pending_tasks{cluster=~"$cluster"} - ) - ||| - ), - - /* - Shard queries - */ - - activeShards: - prometheusQuery.new( - '$' + variables.datasource.name, - ||| - sum( - elasticsearch_cluster_health_active_shards{cluster=~"$cluster"} - ) - ||| - ), - - activePrimaryShards: - prometheusQuery.new( - '$' + variables.datasource.name, - ||| - sum( - elasticsearch_cluster_health_active_primary_shards{cluster=~"$cluster"} - ) - ||| - ), - - initializingShards: - prometheusQuery.new( - '$' + variables.datasource.name, - ||| - sum( - elasticsearch_cluster_health_initializing_shards{cluster=~"$cluster"} - ) - ||| - ), - - reloactingShards: - prometheusQuery.new( - '$' + variables.datasource.name, - ||| - sum( - elasticsearch_cluster_health_reloacting_shards{cluster=~"$cluster"} - ) - ||| - ), - - unassignedShards: - prometheusQuery.new( - '$' + variables.datasource.name, - ||| - sum( - elasticsearch_cluster_health_unassigned_shards{cluster=~"$cluster"} - ) - ||| - ), - - delayedUnassignedShards: - prometheusQuery.new( - '$' + variables.datasource.name, - ||| - sum( - elasticsearch_cluster_health_delayed_unassigned_shards{cluster=~"$cluster"} - ) - ||| - ), - - /* - Documents queries - */ - - indexedDocuments: - prometheusQuery.new( - '$' + variables.datasource.name, - ||| - elasticsearch_indices_docs{cluster=~"$cluster"} - ||| - ), -} +(import './queries/general.libsonnet') + +(import './queries/shard.libsonnet') + +(import './queries/document.libsonnet') + +(import './queries/memory.libsonnet') + +(import './queries/threads.libsonnet') + +(import './queries/network.libsonnet') diff --git a/elasticsearch_mixin/dashboards/queries/document.libsonnet b/elasticsearch_mixin/dashboards/queries/document.libsonnet new file mode 100644 index 00000000..369ce885 --- /dev/null +++ b/elasticsearch_mixin/dashboards/queries/document.libsonnet @@ -0,0 +1,50 @@ +local g = import '../g.libsonnet'; +local prometheusQuery = g.query.prometheus; + +local variables = import '../variables.libsonnet'; + +{ + indexedDocuments: + prometheusQuery.new( + '$' + variables.datasource.name, + ||| + elasticsearch_indices_docs{cluster=~"$cluster"} + ||| + ), + + indexSize: + prometheusQuery.new( + '$' + variables.datasource.name, + ||| + elasticsearch_indices_store_size_bytes{cluster=~"$cluster"} + ||| + ), + + indexRate: + prometheusQuery.new( + '$' + variables.datasource.name, + ||| + rate(elasticsearch_indices_indexing_index_total{cluster=~"$cluster"}[$__rate_interval]) + ||| + ) + + prometheusQuery.withLegendFormat('{{name}}'), + + queryRate: + prometheusQuery.new( + '$' + variables.datasource.name, + ||| + rate(elasticsearch_indices_search_query_total{cluster=~"$cluster"}[$__rate_interval]) + ||| + ) + + prometheusQuery.withLegendFormat('{{name}}'), + + queueCount: + prometheusQuery.new( + '$' + variables.datasource.name, + ||| + sum(elasticsearch_thread_pool_queue_count{cluster=~"$cluster",type!="management"}) by (type) + ||| + ) + + prometheusQuery.withLegendFormat('{{type}}'), + +} diff --git a/elasticsearch_mixin/dashboards/queries/general.libsonnet b/elasticsearch_mixin/dashboards/queries/general.libsonnet new file mode 100644 index 00000000..053207ac --- /dev/null +++ b/elasticsearch_mixin/dashboards/queries/general.libsonnet @@ -0,0 +1,35 @@ +local g = import '../g.libsonnet'; +local prometheusQuery = g.query.prometheus; + +local variables = import '../variables.libsonnet'; + +{ + runningNodes: + prometheusQuery.new( + '$' + variables.datasource.name, + ||| + sum( + elasticsearch_cluster_health_number_of_nodes{cluster=~"$cluster"} + ) + ||| + ), + dataNodes: + prometheusQuery.new( + '$' + variables.datasource.name, + ||| + sum( + elasticsearch_cluster_health_number_of_data_nodes{cluster=~"$cluster"} + ) + ||| + ), + + pendingTasks: + prometheusQuery.new( + '$' + variables.datasource.name, + ||| + sum( + elasticsearch_cluster_health_number_of_pending_tasks{cluster=~"$cluster"} + ) + ||| + ), +} diff --git a/elasticsearch_mixin/dashboards/queries/memory.libsonnet b/elasticsearch_mixin/dashboards/queries/memory.libsonnet new file mode 100644 index 00000000..59e8d274 --- /dev/null +++ b/elasticsearch_mixin/dashboards/queries/memory.libsonnet @@ -0,0 +1,47 @@ +local g = import '../g.libsonnet'; +local prometheusQuery = g.query.prometheus; + +local variables = import '../variables.libsonnet'; + +{ + memoryUsage: + prometheusQuery.new( + '$' + variables.datasource.name, + ||| + elasticsearch_jvm_memory_used_bytes{cluster=~"$cluster"} + ||| + ) + + prometheusQuery.withLegendFormat('{{name}} {{area}}'), + + memoryUsageAverage15: + prometheusQuery.new( + '$' + variables.datasource.name, + ||| + avg_over_time( + elasticsearch_jvm_memory_used_bytes{cluster=~"$cluster"}[15m] + ) / + elasticsearch_jvm_memory_max_bytes{cluster=~"$cluster"} + ||| + ) + + prometheusQuery.withLegendFormat('{{name}} {{area}}'), + + memoryMax: + prometheusQuery.new( + '$' + variables.datasource.name, + ||| + elasticsearch_jvm_memory_max_bytes{cluster=~"$cluster"} + ||| + ) + + prometheusQuery.withLegendFormat('{{name}} {{area}}'), + + gcSeconds: + prometheusQuery.new( + '$' + variables.datasource.name, + ||| + rate( + elasticsearch_jvm_gc_collection_seconds_sum{cluster=~"$cluster"}[$__rate_interval] + ) + ||| + ) + + prometheusQuery.withLegendFormat('{{name}} {{gc}}'), +} diff --git a/elasticsearch_mixin/dashboards/queries/network.libsonnet b/elasticsearch_mixin/dashboards/queries/network.libsonnet new file mode 100644 index 00000000..76aab2b5 --- /dev/null +++ b/elasticsearch_mixin/dashboards/queries/network.libsonnet @@ -0,0 +1,28 @@ +local g = import '../g.libsonnet'; +local prometheusQuery = g.query.prometheus; + +local variables = import '../variables.libsonnet'; + +{ + transportTXRate: + prometheusQuery.new( + '$' + variables.datasource.name, + ||| + rate( + elasticsearch_transport_rx_size_bytes_total{cluster=~"$cluster"}[$__rate_interval] + ) + ||| + ) + + prometheusQuery.withLegendFormat('{{name}} TX'), + + transportRXRate: + prometheusQuery.new( + '$' + variables.datasource.name, + ||| + rate( + elasticsearch_transport_tx_size_bytes_total{cluster=~"$cluster"}[$__rate_interval] + ) + ||| + ) + + prometheusQuery.withLegendFormat('{{name}} RX'), +} diff --git a/elasticsearch_mixin/dashboards/queries/shard.libsonnet b/elasticsearch_mixin/dashboards/queries/shard.libsonnet new file mode 100644 index 00000000..6bebb4e6 --- /dev/null +++ b/elasticsearch_mixin/dashboards/queries/shard.libsonnet @@ -0,0 +1,66 @@ +local g = import '../g.libsonnet'; +local prometheusQuery = g.query.prometheus; + +local variables = import '../variables.libsonnet'; + +{ + activeShards: + prometheusQuery.new( + '$' + variables.datasource.name, + ||| + sum( + elasticsearch_cluster_health_active_shards{cluster=~"$cluster"} + ) + ||| + ), + + activePrimaryShards: + prometheusQuery.new( + '$' + variables.datasource.name, + ||| + sum( + elasticsearch_cluster_health_active_primary_shards{cluster=~"$cluster"} + ) + ||| + ), + + initializingShards: + prometheusQuery.new( + '$' + variables.datasource.name, + ||| + sum( + elasticsearch_cluster_health_initializing_shards{cluster=~"$cluster"} + ) + ||| + ), + + reloactingShards: + prometheusQuery.new( + '$' + variables.datasource.name, + ||| + sum( + elasticsearch_cluster_health_reloacting_shards{cluster=~"$cluster"} + ) + ||| + ), + + unassignedShards: + prometheusQuery.new( + '$' + variables.datasource.name, + ||| + sum( + elasticsearch_cluster_health_unassigned_shards{cluster=~"$cluster"} + ) + ||| + ), + + delayedUnassignedShards: + prometheusQuery.new( + '$' + variables.datasource.name, + ||| + sum( + elasticsearch_cluster_health_delayed_unassigned_shards{cluster=~"$cluster"} + ) + ||| + ), +} diff --git a/elasticsearch_mixin/dashboards/queries/threads.libsonnet b/elasticsearch_mixin/dashboards/queries/threads.libsonnet new file mode 100644 index 00000000..1fdb3e63 --- /dev/null +++ b/elasticsearch_mixin/dashboards/queries/threads.libsonnet @@ -0,0 +1,24 @@ +local g = import '../g.libsonnet'; +local prometheusQuery = g.query.prometheus; + +local variables = import '../variables.libsonnet'; + +{ + threadPoolActive: + prometheusQuery.new( + '$' + variables.datasource.name, + ||| + elasticsearch_thread_pool_active_count{cluster=~"$cluster"} + ||| + ) + + prometheusQuery.withLegendFormat('{{type}}'), + + threadPoolRejections: + prometheusQuery.new( + '$' + variables.datasource.name, + ||| + elasticsearch_thread_pool_rejected_count{cluster=~"$cluster"} + ||| + ) + + prometheusQuery.withLegendFormat('{{name}} {{type}}'), +} From 1bdb499a33c6868b5dc317ea73e0eef7a2d5dc8e Mon Sep 17 00:00:00 2001 From: Joe Adams Date: Sun, 14 Jul 2024 03:27:09 -0400 Subject: [PATCH 04/12] Rename mixin, add CI Signed-off-by: Joe Adams --- .github/workflows/mixin.yml | 25 + .../README.md | 19 +- .../config.libsonnet | 0 .../dashboards.jsonnet | 2 +- .../dashboards/cluster.libsonnet | 0 .../dashboards/dashboards.libsonnet | 0 elasticsearch-mixin/dashboards/g.libsonnet | 1 + .../dashboards/panels.libsonnet | 0 .../dashboards/queries.libsonnet | 0 .../dashboards/queries/document.libsonnet | 0 .../dashboards/queries/general.libsonnet | 0 .../dashboards/queries/memory.libsonnet | 0 .../dashboards/queries/network.libsonnet | 0 .../dashboards/queries/shard.libsonnet | 0 .../dashboards/queries/threads.libsonnet | 0 .../dashboards/variables.libsonnet | 2 +- .../jsonnetfile.json | 0 .../jsonnetfile.lock.json | 0 .../mixin.libsonnet | 0 elasticsearch-mixin/out.json | 563 ++++++++++++++++++ elasticsearch_mixin/dashboards/g.libsonnet | 1 - scripts/lint-jsonnet.sh | 13 + 22 files changed, 622 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/mixin.yml rename {elasticsearch_mixin => elasticsearch-mixin}/README.md (55%) rename {elasticsearch_mixin => elasticsearch-mixin}/config.libsonnet (100%) rename {elasticsearch_mixin => elasticsearch-mixin}/dashboards.jsonnet (95%) rename {elasticsearch_mixin => elasticsearch-mixin}/dashboards/cluster.libsonnet (100%) rename {elasticsearch_mixin => elasticsearch-mixin}/dashboards/dashboards.libsonnet (100%) create mode 100644 elasticsearch-mixin/dashboards/g.libsonnet rename {elasticsearch_mixin => elasticsearch-mixin}/dashboards/panels.libsonnet (100%) rename {elasticsearch_mixin => elasticsearch-mixin}/dashboards/queries.libsonnet (100%) rename {elasticsearch_mixin => elasticsearch-mixin}/dashboards/queries/document.libsonnet (100%) rename {elasticsearch_mixin => elasticsearch-mixin}/dashboards/queries/general.libsonnet (100%) rename {elasticsearch_mixin => elasticsearch-mixin}/dashboards/queries/memory.libsonnet (100%) rename {elasticsearch_mixin => elasticsearch-mixin}/dashboards/queries/network.libsonnet (100%) rename {elasticsearch_mixin => elasticsearch-mixin}/dashboards/queries/shard.libsonnet (100%) rename {elasticsearch_mixin => elasticsearch-mixin}/dashboards/queries/threads.libsonnet (100%) rename {elasticsearch_mixin => elasticsearch-mixin}/dashboards/variables.libsonnet (98%) rename {elasticsearch_mixin => elasticsearch-mixin}/jsonnetfile.json (100%) rename {elasticsearch_mixin => elasticsearch-mixin}/jsonnetfile.lock.json (100%) rename {elasticsearch_mixin => elasticsearch-mixin}/mixin.libsonnet (100%) create mode 100644 elasticsearch-mixin/out.json delete mode 100644 elasticsearch_mixin/dashboards/g.libsonnet create mode 100755 scripts/lint-jsonnet.sh diff --git a/.github/workflows/mixin.yml b/.github/workflows/mixin.yml new file mode 100644 index 00000000..3562335f --- /dev/null +++ b/.github/workflows/mixin.yml @@ -0,0 +1,25 @@ +--- +name: mixin +on: + pull_request: + paths: + - "elasticsearch-mixin/**" + +jobs: + lint: + name: lint + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: 1.22.5 + - name: Install dependencies + run: | + go install github.com/google/go-jsonnet/cmd/jsonnet@v0.20.0 + go install github.com/google/go-jsonnet/cmd/jsonnetfmt@v0.20.0 + go install github.com/google/go-jsonnet/cmd/jsonnet-lint@v0.20.0 + - name: Lint + run: bash ./scripts/lint-jsonnet.sh diff --git a/elasticsearch_mixin/README.md b/elasticsearch-mixin/README.md similarity index 55% rename from elasticsearch_mixin/README.md rename to elasticsearch-mixin/README.md index dec45492..ef290faa 100644 --- a/elasticsearch_mixin/README.md +++ b/elasticsearch-mixin/README.md @@ -5,16 +5,33 @@ This is a mixin for the elasticsearch_exporter to define dashboards, alerts, and Good example of upstream mixin for reference: https://github.com/kubernetes-monitoring/kubernetes-mixin +docker-compose +- docker-compose exec elasticsearch bash + - bin/elasticsearch-reset-password -u elastic -f +- login to grafana +- add prometheus datasource (http://prometheus:9090) +- http://127.0.0.1:3000 +- http://127.0.0.1:9090/targets?search= +- http://127.0.0.1:9114/metrics + ## Development ### JSONNET +https://jsonnet.org/ + ```go install github.com/google/go-jsonnet/cmd/jsonnet@latest``` ### JSONNET BUNDLER jsonnet bundler is a package manager for jsonnet -```go install -a github.com/jsonnet-bundler/jsonnet-bundler/cmd/jb@latest``` +https://github.com/jsonnet-bundler/jsonnet-bundler +```go install -a github.com/jsonnet-bundler/jsonnet-bundler/cmd/jb@latest``` ### Grafonnet +Grafana libraries for jsonnet: https://grafana.github.io/grafonnet/ + ```jb install github.com/grafana/grafonnet/gen/grafonnet-latest@main``` + +validate +go install github.com/grafana/dashboard-linter@latest diff --git a/elasticsearch_mixin/config.libsonnet b/elasticsearch-mixin/config.libsonnet similarity index 100% rename from elasticsearch_mixin/config.libsonnet rename to elasticsearch-mixin/config.libsonnet diff --git a/elasticsearch_mixin/dashboards.jsonnet b/elasticsearch-mixin/dashboards.jsonnet similarity index 95% rename from elasticsearch_mixin/dashboards.jsonnet rename to elasticsearch-mixin/dashboards.jsonnet index d606bd6c..9a03074e 100644 --- a/elasticsearch_mixin/dashboards.jsonnet +++ b/elasticsearch-mixin/dashboards.jsonnet @@ -1,3 +1,3 @@ local dashboards = (import 'mixin.libsonnet').grafanaDashboards; -{ [name]: dashboards[name] for name in std.objectFields(dashboards)} +{ [name]: dashboards[name] for name in std.objectFields(dashboards) } diff --git a/elasticsearch_mixin/dashboards/cluster.libsonnet b/elasticsearch-mixin/dashboards/cluster.libsonnet similarity index 100% rename from elasticsearch_mixin/dashboards/cluster.libsonnet rename to elasticsearch-mixin/dashboards/cluster.libsonnet diff --git a/elasticsearch_mixin/dashboards/dashboards.libsonnet b/elasticsearch-mixin/dashboards/dashboards.libsonnet similarity index 100% rename from elasticsearch_mixin/dashboards/dashboards.libsonnet rename to elasticsearch-mixin/dashboards/dashboards.libsonnet diff --git a/elasticsearch-mixin/dashboards/g.libsonnet b/elasticsearch-mixin/dashboards/g.libsonnet new file mode 100644 index 00000000..69aac830 --- /dev/null +++ b/elasticsearch-mixin/dashboards/g.libsonnet @@ -0,0 +1 @@ +import 'github.com/grafana/grafonnet/gen/grafonnet-latest/main.libsonnet' diff --git a/elasticsearch_mixin/dashboards/panels.libsonnet b/elasticsearch-mixin/dashboards/panels.libsonnet similarity index 100% rename from elasticsearch_mixin/dashboards/panels.libsonnet rename to elasticsearch-mixin/dashboards/panels.libsonnet diff --git a/elasticsearch_mixin/dashboards/queries.libsonnet b/elasticsearch-mixin/dashboards/queries.libsonnet similarity index 100% rename from elasticsearch_mixin/dashboards/queries.libsonnet rename to elasticsearch-mixin/dashboards/queries.libsonnet diff --git a/elasticsearch_mixin/dashboards/queries/document.libsonnet b/elasticsearch-mixin/dashboards/queries/document.libsonnet similarity index 100% rename from elasticsearch_mixin/dashboards/queries/document.libsonnet rename to elasticsearch-mixin/dashboards/queries/document.libsonnet diff --git a/elasticsearch_mixin/dashboards/queries/general.libsonnet b/elasticsearch-mixin/dashboards/queries/general.libsonnet similarity index 100% rename from elasticsearch_mixin/dashboards/queries/general.libsonnet rename to elasticsearch-mixin/dashboards/queries/general.libsonnet diff --git a/elasticsearch_mixin/dashboards/queries/memory.libsonnet b/elasticsearch-mixin/dashboards/queries/memory.libsonnet similarity index 100% rename from elasticsearch_mixin/dashboards/queries/memory.libsonnet rename to elasticsearch-mixin/dashboards/queries/memory.libsonnet diff --git a/elasticsearch_mixin/dashboards/queries/network.libsonnet b/elasticsearch-mixin/dashboards/queries/network.libsonnet similarity index 100% rename from elasticsearch_mixin/dashboards/queries/network.libsonnet rename to elasticsearch-mixin/dashboards/queries/network.libsonnet diff --git a/elasticsearch_mixin/dashboards/queries/shard.libsonnet b/elasticsearch-mixin/dashboards/queries/shard.libsonnet similarity index 100% rename from elasticsearch_mixin/dashboards/queries/shard.libsonnet rename to elasticsearch-mixin/dashboards/queries/shard.libsonnet diff --git a/elasticsearch_mixin/dashboards/queries/threads.libsonnet b/elasticsearch-mixin/dashboards/queries/threads.libsonnet similarity index 100% rename from elasticsearch_mixin/dashboards/queries/threads.libsonnet rename to elasticsearch-mixin/dashboards/queries/threads.libsonnet diff --git a/elasticsearch_mixin/dashboards/variables.libsonnet b/elasticsearch-mixin/dashboards/variables.libsonnet similarity index 98% rename from elasticsearch_mixin/dashboards/variables.libsonnet rename to elasticsearch-mixin/dashboards/variables.libsonnet index c8d5715a..53233a23 100644 --- a/elasticsearch_mixin/dashboards/variables.libsonnet +++ b/elasticsearch-mixin/dashboards/variables.libsonnet @@ -11,5 +11,5 @@ local var = g.dashboard.variable; + var.query.queryTypes.withLabelValues( 'cluster', 'elasticsearch_cluster_health_status', - ) + ), } diff --git a/elasticsearch_mixin/jsonnetfile.json b/elasticsearch-mixin/jsonnetfile.json similarity index 100% rename from elasticsearch_mixin/jsonnetfile.json rename to elasticsearch-mixin/jsonnetfile.json diff --git a/elasticsearch_mixin/jsonnetfile.lock.json b/elasticsearch-mixin/jsonnetfile.lock.json similarity index 100% rename from elasticsearch_mixin/jsonnetfile.lock.json rename to elasticsearch-mixin/jsonnetfile.lock.json diff --git a/elasticsearch_mixin/mixin.libsonnet b/elasticsearch-mixin/mixin.libsonnet similarity index 100% rename from elasticsearch_mixin/mixin.libsonnet rename to elasticsearch-mixin/mixin.libsonnet diff --git a/elasticsearch-mixin/out.json b/elasticsearch-mixin/out.json new file mode 100644 index 00000000..64268c6a --- /dev/null +++ b/elasticsearch-mixin/out.json @@ -0,0 +1,563 @@ +{ + "cluster.json": { + "graphTooltip": 1, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0 + }, + "id": 1, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "id": 2, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "sum(\n elasticsearch_cluster_health_number_of_nodes{cluster=~\"$cluster\"}\n)\n" + } + ], + "title": "Nodes", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "id": 3, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "sum(\n elasticsearch_cluster_health_number_of_data_nodes{cluster=~\"$cluster\"}\n)\n" + } + ], + "title": "Data Nodes", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "id": 4, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "sum(\n elasticsearch_cluster_health_number_of_pending_tasks{cluster=~\"$cluster\"}\n)\n" + } + ], + "title": "Pending Tasks", + "type": "stat" + } + ], + "title": "Overview", + "type": "row" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0 + }, + "id": 5, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "id": 6, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "sum(\n elasticsearch_cluster_health_active_shards{cluster=~\"$cluster\"}\n)\n" + } + ], + "title": "Active", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "id": 7, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "sum(\n elasticsearch_cluster_health_active_primary_shards{cluster=~\"$cluster\"}\n)\n" + } + ], + "title": "Active Primary", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "id": 8, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "sum(\n elasticsearch_cluster_health_initializing_shards{cluster=~\"$cluster\"}\n)\n" + } + ], + "title": "Initializing", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "id": 9, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "sum(\n elasticsearch_cluster_health_reloacting_shards{cluster=~\"$cluster\"}\n)\n" + } + ], + "title": "Relocating", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "id": 10, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "sum(\n elasticsearch_cluster_health_unassigned_shards{cluster=~\"$cluster\"}\n)\n" + } + ], + "title": "Unassigned", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "id": 11, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "sum(\n elasticsearch_cluster_health_delayed_unassigned_shards{cluster=~\"$cluster\"}\n)\n" + } + ], + "title": "DelayedUnassigned", + "type": "stat" + } + ], + "title": "Shards", + "type": "row" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0 + }, + "id": 12, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "id": 13, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "elasticsearch_indices_docs{cluster=~\"$cluster\"}\n" + } + ], + "title": "Indexed Documents", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "unit": "bytes" + } + }, + "id": 14, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "elasticsearch_indices_store_size_bytes{cluster=~\"$cluster\"}\n" + } + ], + "title": "Index Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "id": 15, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "rate(elasticsearch_indices_indexing_index_total{cluster=~\"$cluster\"}[$__rate_interval])\n", + "legendFormat": "{{name}}" + } + ], + "title": "Index Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "id": 16, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "rate(elasticsearch_indices_search_query_total{cluster=~\"$cluster\"}[$__rate_interval])\n", + "legendFormat": "{{name}}" + } + ], + "title": "Query Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "id": 17, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "sum(elasticsearch_thread_pool_queue_count{cluster=~\"$cluster\",type!=\"management\"}) by (type)\n", + "legendFormat": "{{type}}" + } + ], + "title": "Queue Count", + "type": "timeseries" + } + ], + "title": "Documents", + "type": "row" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0 + }, + "id": 18, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "unit": "bytes" + } + }, + "id": 19, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "elasticsearch_jvm_memory_used_bytes{cluster=~\"$cluster\"}\n", + "legendFormat": "{{name}} {{area}}" + } + ], + "title": "Memory Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "max": 1, + "min": 0, + "unit": "percentunit" + } + }, + "id": 20, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "avg_over_time(\n elasticsearch_jvm_memory_used_bytes{cluster=~\"$cluster\"}[15m]\n) /\nelasticsearch_jvm_memory_max_bytes{cluster=~\"$cluster\"}\n", + "legendFormat": "{{name}} {{area}}" + } + ], + "title": "Memory 15m Avg", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "unit": "bytes" + } + }, + "id": 21, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "elasticsearch_jvm_memory_max_bytes{cluster=~\"$cluster\"}\n", + "legendFormat": "{{name}} {{area}}" + } + ], + "title": "Memory Max", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "unit": "s" + } + }, + "id": 22, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "rate(\n elasticsearch_jvm_gc_collection_seconds_sum{cluster=~\"$cluster\"}[$__rate_interval]\n)\n", + "legendFormat": "{{name}} {{gc}}" + } + ], + "title": "GC Rate", + "type": "timeseries" + } + ], + "title": "Memory", + "type": "row" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0 + }, + "id": 23, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "id": 24, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "elasticsearch_thread_pool_active_count{cluster=~\"$cluster\"}\n", + "legendFormat": "{{type}}" + } + ], + "title": "Thread Pools", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "id": 25, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "elasticsearch_thread_pool_rejected_count{cluster=~\"$cluster\"}\n", + "legendFormat": "{{name}} {{type}}" + } + ], + "title": "Thread Pool Rejections", + "type": "timeseries" + } + ], + "title": "Threads", + "type": "row" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0 + }, + "id": 26, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "unit": "bytes" + } + }, + "id": 27, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "rate(\n elasticsearch_transport_rx_size_bytes_total{cluster=~\"$cluster\"}[$__rate_interval]\n)\n", + "legendFormat": "{{name}} TX" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "rate(\n elasticsearch_transport_tx_size_bytes_total{cluster=~\"$cluster\"}[$__rate_interval]\n)\n", + "legendFormat": "{{name}} RX" + } + ], + "title": "Transport Rate", + "type": "timeseries" + } + ], + "title": "Network", + "type": "row" + } + ], + "refresh": "1m", + "schemaVersion": 36, + "tags": [ + "elasticsearch-exporter-mixin" + ], + "templating": { + "list": [ + { + "name": "datasource", + "query": "prometheus", + "type": "datasource" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "name": "cluster", + "query": "label_values(elasticsearch_cluster_health_status, cluster)", + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "utc", + "title": "Elasticsearch Exporter / Cluster" + } +} diff --git a/elasticsearch_mixin/dashboards/g.libsonnet b/elasticsearch_mixin/dashboards/g.libsonnet deleted file mode 100644 index 71fdb755..00000000 --- a/elasticsearch_mixin/dashboards/g.libsonnet +++ /dev/null @@ -1 +0,0 @@ -import "github.com/grafana/grafonnet/gen/grafonnet-latest/main.libsonnet" diff --git a/scripts/lint-jsonnet.sh b/scripts/lint-jsonnet.sh new file mode 100755 index 00000000..74348bf5 --- /dev/null +++ b/scripts/lint-jsonnet.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +# Run lint on all jsonnet files in the repository +RESULT=0; +for f in $(find . -name 'vendor' -prune -o -name '*.libsonnet' -print -o -name '*.jsonnet' -print); do + # jsonnet fmt -i "$$f" + echo "Linting ${f}" + jsonnetfmt -- "${f}" | diff -u "${f}" - + RESULT=$((RESULT+$?)) +done + +echo "Linting complete" +exit $RESULT From 7619b3a4a4d363ed4d1105ef379847bdfed6a992 Mon Sep 17 00:00:00 2001 From: Joe Adams Date: Sun, 14 Jul 2024 03:44:24 -0400 Subject: [PATCH 05/12] Add CI to verify that mixin is compiled Signed-off-by: Joe Adams --- .github/workflows/mixin.yml | 8 +- elasticsearch-mixin/out.json | 563 ----------------------------------- scripts/compile-mixin.sh | 8 + 3 files changed, 14 insertions(+), 565 deletions(-) delete mode 100644 elasticsearch-mixin/out.json create mode 100755 scripts/compile-mixin.sh diff --git a/.github/workflows/mixin.yml b/.github/workflows/mixin.yml index 3562335f..75c77c45 100644 --- a/.github/workflows/mixin.yml +++ b/.github/workflows/mixin.yml @@ -6,8 +6,8 @@ on: - "elasticsearch-mixin/**" jobs: - lint: - name: lint + check-mixin: + name: check runs-on: ubuntu-latest steps: - name: Checkout repository @@ -23,3 +23,7 @@ jobs: go install github.com/google/go-jsonnet/cmd/jsonnet-lint@v0.20.0 - name: Lint run: bash ./scripts/lint-jsonnet.sh + - name: Compile mixin + run: bash ./scripts/compile-mixin.sh + - name: Verify compiled mixin matches repo + run: git diff --exit-code -- elasticsearch-mixin diff --git a/elasticsearch-mixin/out.json b/elasticsearch-mixin/out.json deleted file mode 100644 index 64268c6a..00000000 --- a/elasticsearch-mixin/out.json +++ /dev/null @@ -1,563 +0,0 @@ -{ - "cluster.json": { - "graphTooltip": 1, - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0 - }, - "id": 1, - "panels": [ - { - "datasource": { - "type": "datasource", - "uid": "-- Mixed --" - }, - "id": 2, - "pluginVersion": "v10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "expr": "sum(\n elasticsearch_cluster_health_number_of_nodes{cluster=~\"$cluster\"}\n)\n" - } - ], - "title": "Nodes", - "type": "stat" - }, - { - "datasource": { - "type": "datasource", - "uid": "-- Mixed --" - }, - "id": 3, - "pluginVersion": "v10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "expr": "sum(\n elasticsearch_cluster_health_number_of_data_nodes{cluster=~\"$cluster\"}\n)\n" - } - ], - "title": "Data Nodes", - "type": "stat" - }, - { - "datasource": { - "type": "datasource", - "uid": "-- Mixed --" - }, - "id": 4, - "pluginVersion": "v10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "expr": "sum(\n elasticsearch_cluster_health_number_of_pending_tasks{cluster=~\"$cluster\"}\n)\n" - } - ], - "title": "Pending Tasks", - "type": "stat" - } - ], - "title": "Overview", - "type": "row" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0 - }, - "id": 5, - "panels": [ - { - "datasource": { - "type": "datasource", - "uid": "-- Mixed --" - }, - "id": 6, - "pluginVersion": "v10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "expr": "sum(\n elasticsearch_cluster_health_active_shards{cluster=~\"$cluster\"}\n)\n" - } - ], - "title": "Active", - "type": "stat" - }, - { - "datasource": { - "type": "datasource", - "uid": "-- Mixed --" - }, - "id": 7, - "pluginVersion": "v10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "expr": "sum(\n elasticsearch_cluster_health_active_primary_shards{cluster=~\"$cluster\"}\n)\n" - } - ], - "title": "Active Primary", - "type": "stat" - }, - { - "datasource": { - "type": "datasource", - "uid": "-- Mixed --" - }, - "id": 8, - "pluginVersion": "v10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "expr": "sum(\n elasticsearch_cluster_health_initializing_shards{cluster=~\"$cluster\"}\n)\n" - } - ], - "title": "Initializing", - "type": "stat" - }, - { - "datasource": { - "type": "datasource", - "uid": "-- Mixed --" - }, - "id": 9, - "pluginVersion": "v10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "expr": "sum(\n elasticsearch_cluster_health_reloacting_shards{cluster=~\"$cluster\"}\n)\n" - } - ], - "title": "Relocating", - "type": "stat" - }, - { - "datasource": { - "type": "datasource", - "uid": "-- Mixed --" - }, - "id": 10, - "pluginVersion": "v10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "expr": "sum(\n elasticsearch_cluster_health_unassigned_shards{cluster=~\"$cluster\"}\n)\n" - } - ], - "title": "Unassigned", - "type": "stat" - }, - { - "datasource": { - "type": "datasource", - "uid": "-- Mixed --" - }, - "id": 11, - "pluginVersion": "v10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "expr": "sum(\n elasticsearch_cluster_health_delayed_unassigned_shards{cluster=~\"$cluster\"}\n)\n" - } - ], - "title": "DelayedUnassigned", - "type": "stat" - } - ], - "title": "Shards", - "type": "row" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0 - }, - "id": 12, - "panels": [ - { - "datasource": { - "type": "datasource", - "uid": "-- Mixed --" - }, - "id": 13, - "pluginVersion": "v10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "expr": "elasticsearch_indices_docs{cluster=~\"$cluster\"}\n" - } - ], - "title": "Indexed Documents", - "type": "timeseries" - }, - { - "datasource": { - "type": "datasource", - "uid": "-- Mixed --" - }, - "fieldConfig": { - "defaults": { - "unit": "bytes" - } - }, - "id": 14, - "pluginVersion": "v10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "expr": "elasticsearch_indices_store_size_bytes{cluster=~\"$cluster\"}\n" - } - ], - "title": "Index Size", - "type": "timeseries" - }, - { - "datasource": { - "type": "datasource", - "uid": "-- Mixed --" - }, - "id": 15, - "pluginVersion": "v10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "expr": "rate(elasticsearch_indices_indexing_index_total{cluster=~\"$cluster\"}[$__rate_interval])\n", - "legendFormat": "{{name}}" - } - ], - "title": "Index Rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "datasource", - "uid": "-- Mixed --" - }, - "id": 16, - "pluginVersion": "v10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "expr": "rate(elasticsearch_indices_search_query_total{cluster=~\"$cluster\"}[$__rate_interval])\n", - "legendFormat": "{{name}}" - } - ], - "title": "Query Rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "datasource", - "uid": "-- Mixed --" - }, - "id": 17, - "pluginVersion": "v10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "expr": "sum(elasticsearch_thread_pool_queue_count{cluster=~\"$cluster\",type!=\"management\"}) by (type)\n", - "legendFormat": "{{type}}" - } - ], - "title": "Queue Count", - "type": "timeseries" - } - ], - "title": "Documents", - "type": "row" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0 - }, - "id": 18, - "panels": [ - { - "datasource": { - "type": "datasource", - "uid": "-- Mixed --" - }, - "fieldConfig": { - "defaults": { - "unit": "bytes" - } - }, - "id": 19, - "pluginVersion": "v10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "expr": "elasticsearch_jvm_memory_used_bytes{cluster=~\"$cluster\"}\n", - "legendFormat": "{{name}} {{area}}" - } - ], - "title": "Memory Usage", - "type": "timeseries" - }, - { - "datasource": { - "type": "datasource", - "uid": "-- Mixed --" - }, - "fieldConfig": { - "defaults": { - "max": 1, - "min": 0, - "unit": "percentunit" - } - }, - "id": 20, - "pluginVersion": "v10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "expr": "avg_over_time(\n elasticsearch_jvm_memory_used_bytes{cluster=~\"$cluster\"}[15m]\n) /\nelasticsearch_jvm_memory_max_bytes{cluster=~\"$cluster\"}\n", - "legendFormat": "{{name}} {{area}}" - } - ], - "title": "Memory 15m Avg", - "type": "timeseries" - }, - { - "datasource": { - "type": "datasource", - "uid": "-- Mixed --" - }, - "fieldConfig": { - "defaults": { - "unit": "bytes" - } - }, - "id": 21, - "pluginVersion": "v10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "expr": "elasticsearch_jvm_memory_max_bytes{cluster=~\"$cluster\"}\n", - "legendFormat": "{{name}} {{area}}" - } - ], - "title": "Memory Max", - "type": "timeseries" - }, - { - "datasource": { - "type": "datasource", - "uid": "-- Mixed --" - }, - "fieldConfig": { - "defaults": { - "unit": "s" - } - }, - "id": 22, - "pluginVersion": "v10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "expr": "rate(\n elasticsearch_jvm_gc_collection_seconds_sum{cluster=~\"$cluster\"}[$__rate_interval]\n)\n", - "legendFormat": "{{name}} {{gc}}" - } - ], - "title": "GC Rate", - "type": "timeseries" - } - ], - "title": "Memory", - "type": "row" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0 - }, - "id": 23, - "panels": [ - { - "datasource": { - "type": "datasource", - "uid": "-- Mixed --" - }, - "id": 24, - "pluginVersion": "v10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "expr": "elasticsearch_thread_pool_active_count{cluster=~\"$cluster\"}\n", - "legendFormat": "{{type}}" - } - ], - "title": "Thread Pools", - "type": "timeseries" - }, - { - "datasource": { - "type": "datasource", - "uid": "-- Mixed --" - }, - "id": 25, - "pluginVersion": "v10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "expr": "elasticsearch_thread_pool_rejected_count{cluster=~\"$cluster\"}\n", - "legendFormat": "{{name}} {{type}}" - } - ], - "title": "Thread Pool Rejections", - "type": "timeseries" - } - ], - "title": "Threads", - "type": "row" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0 - }, - "id": 26, - "panels": [ - { - "datasource": { - "type": "datasource", - "uid": "-- Mixed --" - }, - "fieldConfig": { - "defaults": { - "unit": "bytes" - } - }, - "id": 27, - "pluginVersion": "v10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "expr": "rate(\n elasticsearch_transport_rx_size_bytes_total{cluster=~\"$cluster\"}[$__rate_interval]\n)\n", - "legendFormat": "{{name}} TX" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "expr": "rate(\n elasticsearch_transport_tx_size_bytes_total{cluster=~\"$cluster\"}[$__rate_interval]\n)\n", - "legendFormat": "{{name}} RX" - } - ], - "title": "Transport Rate", - "type": "timeseries" - } - ], - "title": "Network", - "type": "row" - } - ], - "refresh": "1m", - "schemaVersion": 36, - "tags": [ - "elasticsearch-exporter-mixin" - ], - "templating": { - "list": [ - { - "name": "datasource", - "query": "prometheus", - "type": "datasource" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "name": "cluster", - "query": "label_values(elasticsearch_cluster_health_status, cluster)", - "type": "query" - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timezone": "utc", - "title": "Elasticsearch Exporter / Cluster" - } -} diff --git a/scripts/compile-mixin.sh b/scripts/compile-mixin.sh new file mode 100755 index 00000000..1cd1f849 --- /dev/null +++ b/scripts/compile-mixin.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +MIXIN_PATH=./elasticsearch-mixin +MIXIN_OUT_PATH=./elasticsearch-mixin/compiled + +rm -rf ${MIXIN_OUT_PATH} && mkdir ${MIXIN_OUT_PATH} +pushd ${MIXIN_PATH} && jb install && popd +mixtool generate all --output-alerts ${MIXIN_OUT_PATH}/alerts.yaml --output-rules ${MIXIN_OUT_PATH}/rules.yaml --directory ${MIXIN_OUT_PATH}/dashboards ${MIXIN_PATH}/mixin.libsonnet From f12127f0f460229443d17537b9a8f1f27fbc50a5 Mon Sep 17 00:00:00 2001 From: Joe Adams Date: Sun, 14 Jul 2024 03:45:54 -0400 Subject: [PATCH 06/12] Add dependencies for compilation Signed-off-by: Joe Adams --- .github/workflows/mixin.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/mixin.yml b/.github/workflows/mixin.yml index 75c77c45..15e1dcce 100644 --- a/.github/workflows/mixin.yml +++ b/.github/workflows/mixin.yml @@ -21,6 +21,8 @@ jobs: go install github.com/google/go-jsonnet/cmd/jsonnet@v0.20.0 go install github.com/google/go-jsonnet/cmd/jsonnetfmt@v0.20.0 go install github.com/google/go-jsonnet/cmd/jsonnet-lint@v0.20.0 + go install github.com/monitoring-mixins/mixtool/cmd/mixtool@16dc166166d91e93475b86b9355a4faed2400c18 + go install github.com/jsonnet-bundler/jsonnet-bundler/cmd/jb@v0.5.1 - name: Lint run: bash ./scripts/lint-jsonnet.sh - name: Compile mixin From 0f45d664600121e8ccfbfccb67b68f459650f06d Mon Sep 17 00:00:00 2001 From: Joe Adams Date: Sun, 14 Jul 2024 03:58:36 -0400 Subject: [PATCH 07/12] Adjust compilation match logic for CI Signed-off-by: Joe Adams --- .github/workflows/mixin.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/mixin.yml b/.github/workflows/mixin.yml index 15e1dcce..3e1bfac0 100644 --- a/.github/workflows/mixin.yml +++ b/.github/workflows/mixin.yml @@ -28,4 +28,7 @@ jobs: - name: Compile mixin run: bash ./scripts/compile-mixin.sh - name: Verify compiled mixin matches repo - run: git diff --exit-code -- elasticsearch-mixin + run: | + git diff --exit-code -- ./elasticsearch-mixin || (echo "Compiled mixin does not match repo" && exit 1) + # Check if there are any new untracked files + test -z $(git status --porcelain) From 4f50c7f9969618f82cdae5463b88f7a897118ad3 Mon Sep 17 00:00:00 2001 From: Joe Adams Date: Sun, 14 Jul 2024 04:15:56 -0400 Subject: [PATCH 08/12] Fix quotes Signed-off-by: Joe Adams --- .github/workflows/mixin.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/mixin.yml b/.github/workflows/mixin.yml index 3e1bfac0..7948354f 100644 --- a/.github/workflows/mixin.yml +++ b/.github/workflows/mixin.yml @@ -31,4 +31,4 @@ jobs: run: | git diff --exit-code -- ./elasticsearch-mixin || (echo "Compiled mixin does not match repo" && exit 1) # Check if there are any new untracked files - test -z $(git status --porcelain) + test -z "$(git status --porcelain)" From f4c19eef5d55c0f47f7b2ae9af40f4fe720d8c82 Mon Sep 17 00:00:00 2001 From: Joe Adams Date: Sun, 14 Jul 2024 04:18:29 -0400 Subject: [PATCH 09/12] Add helpful error output Signed-off-by: Joe Adams --- .github/workflows/mixin.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/mixin.yml b/.github/workflows/mixin.yml index 7948354f..34a0fa54 100644 --- a/.github/workflows/mixin.yml +++ b/.github/workflows/mixin.yml @@ -31,4 +31,4 @@ jobs: run: | git diff --exit-code -- ./elasticsearch-mixin || (echo "Compiled mixin does not match repo" && exit 1) # Check if there are any new untracked files - test -z "$(git status --porcelain)" + test -z "$(git status --porcelain)" || (echo "Untracked files found" && exit 1) From 1674b8178eb9e4e452a7072b4eea0703e6af58ad Mon Sep 17 00:00:00 2001 From: Joe Adams Date: Sun, 14 Jul 2024 04:45:02 -0400 Subject: [PATCH 10/12] Add compiled mixin artifacts to repo Signed-off-by: Joe Adams --- .github/workflows/mixin.yml | 2 +- elasticsearch-mixin/compiled/alerts.yaml | 1 + .../compiled/dashboards/cluster.json | 687 ++++++++++++++++++ elasticsearch-mixin/compiled/rules.yaml | 1 + 4 files changed, 690 insertions(+), 1 deletion(-) create mode 100644 elasticsearch-mixin/compiled/alerts.yaml create mode 100644 elasticsearch-mixin/compiled/dashboards/cluster.json create mode 100644 elasticsearch-mixin/compiled/rules.yaml diff --git a/.github/workflows/mixin.yml b/.github/workflows/mixin.yml index 34a0fa54..2c282069 100644 --- a/.github/workflows/mixin.yml +++ b/.github/workflows/mixin.yml @@ -31,4 +31,4 @@ jobs: run: | git diff --exit-code -- ./elasticsearch-mixin || (echo "Compiled mixin does not match repo" && exit 1) # Check if there are any new untracked files - test -z "$(git status --porcelain)" || (echo "Untracked files found" && exit 1) + test -z "$(git status --porcelain)" || (echo "Untracked files found, please run ./scripts/compile-mixin.sh" && exit 1) diff --git a/elasticsearch-mixin/compiled/alerts.yaml b/elasticsearch-mixin/compiled/alerts.yaml new file mode 100644 index 00000000..0967ef42 --- /dev/null +++ b/elasticsearch-mixin/compiled/alerts.yaml @@ -0,0 +1 @@ +{} diff --git a/elasticsearch-mixin/compiled/dashboards/cluster.json b/elasticsearch-mixin/compiled/dashboards/cluster.json new file mode 100644 index 00000000..bc9399a3 --- /dev/null +++ b/elasticsearch-mixin/compiled/dashboards/cluster.json @@ -0,0 +1,687 @@ +{ + "graphTooltip": 1, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [ ], + "title": "Overview", + "type": "row" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 2, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "sum(\n elasticsearch_cluster_health_number_of_nodes{cluster=~\"$cluster\"}\n)\n" + } + ], + "title": "Nodes", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 1 + }, + "id": 3, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "sum(\n elasticsearch_cluster_health_number_of_data_nodes{cluster=~\"$cluster\"}\n)\n" + } + ], + "title": "Data Nodes", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 1 + }, + "id": 4, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "sum(\n elasticsearch_cluster_health_number_of_pending_tasks{cluster=~\"$cluster\"}\n)\n" + } + ], + "title": "Pending Tasks", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 4 + }, + "id": 5, + "panels": [ ], + "title": "Shards", + "type": "row" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 5 + }, + "id": 6, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "sum(\n elasticsearch_cluster_health_active_shards{cluster=~\"$cluster\"}\n)\n" + } + ], + "title": "Active", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 5 + }, + "id": 7, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "sum(\n elasticsearch_cluster_health_active_primary_shards{cluster=~\"$cluster\"}\n)\n" + } + ], + "title": "Active Primary", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 5 + }, + "id": 8, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "sum(\n elasticsearch_cluster_health_initializing_shards{cluster=~\"$cluster\"}\n)\n" + } + ], + "title": "Initializing", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 12, + "y": 5 + }, + "id": 9, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "sum(\n elasticsearch_cluster_health_reloacting_shards{cluster=~\"$cluster\"}\n)\n" + } + ], + "title": "Relocating", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 16, + "y": 5 + }, + "id": 10, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "sum(\n elasticsearch_cluster_health_unassigned_shards{cluster=~\"$cluster\"}\n)\n" + } + ], + "title": "Unassigned", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 5 + }, + "id": 11, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "sum(\n elasticsearch_cluster_health_delayed_unassigned_shards{cluster=~\"$cluster\"}\n)\n" + } + ], + "title": "DelayedUnassigned", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 12, + "panels": [ ], + "title": "Documents", + "type": "row" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 9 + }, + "id": 13, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "elasticsearch_indices_docs{cluster=~\"$cluster\"}\n" + } + ], + "title": "Indexed Documents", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "unit": "bytes" + } + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 9 + }, + "id": 14, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "elasticsearch_indices_store_size_bytes{cluster=~\"$cluster\"}\n" + } + ], + "title": "Index Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 9 + }, + "id": 15, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "rate(elasticsearch_indices_indexing_index_total{cluster=~\"$cluster\"}[$__rate_interval])\n", + "legendFormat": "{{name}}" + } + ], + "title": "Index Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 12, + "y": 9 + }, + "id": 16, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "rate(elasticsearch_indices_search_query_total{cluster=~\"$cluster\"}[$__rate_interval])\n", + "legendFormat": "{{name}}" + } + ], + "title": "Query Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 16, + "y": 9 + }, + "id": 17, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "sum(elasticsearch_thread_pool_queue_count{cluster=~\"$cluster\",type!=\"management\"}) by (type)\n", + "legendFormat": "{{type}}" + } + ], + "title": "Queue Count", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 18, + "panels": [ ], + "title": "Memory", + "type": "row" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "unit": "bytes" + } + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 13 + }, + "id": 19, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "elasticsearch_jvm_memory_used_bytes{cluster=~\"$cluster\"}\n", + "legendFormat": "{{name}} {{area}}" + } + ], + "title": "Memory Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "max": 1, + "min": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 13 + }, + "id": 20, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "avg_over_time(\n elasticsearch_jvm_memory_used_bytes{cluster=~\"$cluster\"}[15m]\n) /\nelasticsearch_jvm_memory_max_bytes{cluster=~\"$cluster\"}\n", + "legendFormat": "{{name}} {{area}}" + } + ], + "title": "Memory 15m Avg", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "unit": "bytes" + } + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 13 + }, + "id": 21, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "elasticsearch_jvm_memory_max_bytes{cluster=~\"$cluster\"}\n", + "legendFormat": "{{name}} {{area}}" + } + ], + "title": "Memory Max", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "unit": "s" + } + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 12, + "y": 13 + }, + "id": 22, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "rate(\n elasticsearch_jvm_gc_collection_seconds_sum{cluster=~\"$cluster\"}[$__rate_interval]\n)\n", + "legendFormat": "{{name}} {{gc}}" + } + ], + "title": "GC Rate", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 23, + "panels": [ ], + "title": "Threads", + "type": "row" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 17 + }, + "id": 24, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "elasticsearch_thread_pool_active_count{cluster=~\"$cluster\"}\n", + "legendFormat": "{{type}}" + } + ], + "title": "Thread Pools", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 17 + }, + "id": 25, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "elasticsearch_thread_pool_rejected_count{cluster=~\"$cluster\"}\n", + "legendFormat": "{{name}} {{type}}" + } + ], + "title": "Thread Pool Rejections", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 26, + "panels": [ ], + "title": "Network", + "type": "row" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "unit": "bytes" + } + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 21 + }, + "id": 27, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "rate(\n elasticsearch_transport_rx_size_bytes_total{cluster=~\"$cluster\"}[$__rate_interval]\n)\n", + "legendFormat": "{{name}} TX" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "rate(\n elasticsearch_transport_tx_size_bytes_total{cluster=~\"$cluster\"}[$__rate_interval]\n)\n", + "legendFormat": "{{name}} RX" + } + ], + "title": "Transport Rate", + "type": "timeseries" + } + ], + "refresh": "1m", + "schemaVersion": 36, + "tags": [ + "elasticsearch-exporter-mixin" + ], + "templating": { + "list": [ + { + "name": "datasource", + "query": "prometheus", + "type": "datasource" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "name": "cluster", + "query": "label_values(elasticsearch_cluster_health_status, cluster)", + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "utc", + "title": "Elasticsearch Exporter / Cluster" + } \ No newline at end of file diff --git a/elasticsearch-mixin/compiled/rules.yaml b/elasticsearch-mixin/compiled/rules.yaml new file mode 100644 index 00000000..0967ef42 --- /dev/null +++ b/elasticsearch-mixin/compiled/rules.yaml @@ -0,0 +1 @@ +{} From d441e67a692f404e3f096ae358c1ee2f9325b7a7 Mon Sep 17 00:00:00 2001 From: Joe Adams Date: Sat, 7 Sep 2024 11:31:21 -0400 Subject: [PATCH 11/12] Update grid layout for for cluster dashboard Signed-off-by: Joe Adams --- .../compiled/dashboards/cluster.json | 120 +++++++++--------- .../dashboards/cluster.libsonnet | 5 +- elasticsearch-mixin/dashboards/util.libsonnet | 66 ++++++++++ 3 files changed, 129 insertions(+), 62 deletions(-) create mode 100644 elasticsearch-mixin/dashboards/util.libsonnet diff --git a/elasticsearch-mixin/compiled/dashboards/cluster.json b/elasticsearch-mixin/compiled/dashboards/cluster.json index bc9399a3..37c919b6 100644 --- a/elasticsearch-mixin/compiled/dashboards/cluster.json +++ b/elasticsearch-mixin/compiled/dashboards/cluster.json @@ -20,8 +20,8 @@ "uid": "-- Mixed --" }, "gridPos": { - "h": 3, - "w": 4, + "h": 4, + "w": 8, "x": 0, "y": 1 }, @@ -45,9 +45,9 @@ "uid": "-- Mixed --" }, "gridPos": { - "h": 3, - "w": 4, - "x": 4, + "h": 4, + "w": 8, + "x": 8, "y": 1 }, "id": 3, @@ -70,9 +70,9 @@ "uid": "-- Mixed --" }, "gridPos": { - "h": 3, - "w": 4, - "x": 8, + "h": 4, + "w": 8, + "x": 16, "y": 1 }, "id": 4, @@ -95,7 +95,7 @@ "h": 1, "w": 24, "x": 0, - "y": 4 + "y": 5 }, "id": 5, "panels": [ ], @@ -108,10 +108,10 @@ "uid": "-- Mixed --" }, "gridPos": { - "h": 3, + "h": 4, "w": 4, "x": 0, - "y": 5 + "y": 6 }, "id": 6, "pluginVersion": "v10.4.0", @@ -133,10 +133,10 @@ "uid": "-- Mixed --" }, "gridPos": { - "h": 3, + "h": 4, "w": 4, "x": 4, - "y": 5 + "y": 6 }, "id": 7, "pluginVersion": "v10.4.0", @@ -158,10 +158,10 @@ "uid": "-- Mixed --" }, "gridPos": { - "h": 3, + "h": 4, "w": 4, "x": 8, - "y": 5 + "y": 6 }, "id": 8, "pluginVersion": "v10.4.0", @@ -183,10 +183,10 @@ "uid": "-- Mixed --" }, "gridPos": { - "h": 3, + "h": 4, "w": 4, "x": 12, - "y": 5 + "y": 6 }, "id": 9, "pluginVersion": "v10.4.0", @@ -208,10 +208,10 @@ "uid": "-- Mixed --" }, "gridPos": { - "h": 3, + "h": 4, "w": 4, "x": 16, - "y": 5 + "y": 6 }, "id": 10, "pluginVersion": "v10.4.0", @@ -233,10 +233,10 @@ "uid": "-- Mixed --" }, "gridPos": { - "h": 3, + "h": 4, "w": 4, "x": 20, - "y": 5 + "y": 6 }, "id": 11, "pluginVersion": "v10.4.0", @@ -258,7 +258,7 @@ "h": 1, "w": 24, "x": 0, - "y": 8 + "y": 10 }, "id": 12, "panels": [ ], @@ -271,10 +271,10 @@ "uid": "-- Mixed --" }, "gridPos": { - "h": 3, + "h": 4, "w": 4, "x": 0, - "y": 9 + "y": 11 }, "id": 13, "pluginVersion": "v10.4.0", @@ -301,10 +301,10 @@ } }, "gridPos": { - "h": 3, + "h": 4, "w": 4, "x": 4, - "y": 9 + "y": 11 }, "id": 14, "pluginVersion": "v10.4.0", @@ -326,10 +326,10 @@ "uid": "-- Mixed --" }, "gridPos": { - "h": 3, + "h": 4, "w": 4, "x": 8, - "y": 9 + "y": 11 }, "id": 15, "pluginVersion": "v10.4.0", @@ -352,10 +352,10 @@ "uid": "-- Mixed --" }, "gridPos": { - "h": 3, + "h": 4, "w": 4, "x": 12, - "y": 9 + "y": 11 }, "id": 16, "pluginVersion": "v10.4.0", @@ -378,10 +378,10 @@ "uid": "-- Mixed --" }, "gridPos": { - "h": 3, + "h": 4, "w": 4, "x": 16, - "y": 9 + "y": 11 }, "id": 17, "pluginVersion": "v10.4.0", @@ -404,7 +404,7 @@ "h": 1, "w": 24, "x": 0, - "y": 12 + "y": 15 }, "id": 18, "panels": [ ], @@ -422,10 +422,10 @@ } }, "gridPos": { - "h": 3, - "w": 4, + "h": 4, + "w": 6, "x": 0, - "y": 13 + "y": 16 }, "id": 19, "pluginVersion": "v10.4.0", @@ -455,10 +455,10 @@ } }, "gridPos": { - "h": 3, - "w": 4, - "x": 4, - "y": 13 + "h": 4, + "w": 6, + "x": 6, + "y": 16 }, "id": 20, "pluginVersion": "v10.4.0", @@ -486,10 +486,10 @@ } }, "gridPos": { - "h": 3, - "w": 4, - "x": 8, - "y": 13 + "h": 4, + "w": 6, + "x": 12, + "y": 16 }, "id": 21, "pluginVersion": "v10.4.0", @@ -517,10 +517,10 @@ } }, "gridPos": { - "h": 3, - "w": 4, - "x": 12, - "y": 13 + "h": 4, + "w": 6, + "x": 18, + "y": 16 }, "id": 22, "pluginVersion": "v10.4.0", @@ -543,7 +543,7 @@ "h": 1, "w": 24, "x": 0, - "y": 16 + "y": 20 }, "id": 23, "panels": [ ], @@ -556,10 +556,10 @@ "uid": "-- Mixed --" }, "gridPos": { - "h": 3, - "w": 4, + "h": 4, + "w": 12, "x": 0, - "y": 17 + "y": 21 }, "id": 24, "pluginVersion": "v10.4.0", @@ -582,10 +582,10 @@ "uid": "-- Mixed --" }, "gridPos": { - "h": 3, - "w": 4, - "x": 4, - "y": 17 + "h": 4, + "w": 12, + "x": 12, + "y": 21 }, "id": 25, "pluginVersion": "v10.4.0", @@ -608,7 +608,7 @@ "h": 1, "w": 24, "x": 0, - "y": 20 + "y": 25 }, "id": 26, "panels": [ ], @@ -626,10 +626,10 @@ } }, "gridPos": { - "h": 3, - "w": 4, + "h": 4, + "w": 24, "x": 0, - "y": 21 + "y": 26 }, "id": 27, "pluginVersion": "v10.4.0", diff --git a/elasticsearch-mixin/dashboards/cluster.libsonnet b/elasticsearch-mixin/dashboards/cluster.libsonnet index 45744279..ac32d75e 100644 --- a/elasticsearch-mixin/dashboards/cluster.libsonnet +++ b/elasticsearch-mixin/dashboards/cluster.libsonnet @@ -6,6 +6,7 @@ local row = g.panel.row; local panels = import './panels.libsonnet'; local queries = import './queries.libsonnet'; local variables = import './variables.libsonnet'; +local util = import './util.libsonnet'; { grafanaDashboards+:: { @@ -20,7 +21,7 @@ local variables = import './variables.libsonnet'; variables.cluster, ]) + dashboard.withPanels( - g.util.grid.makeGrid([ + util.makeGrid([ row.new('Overview') + row.withPanels([ panels.stat.nodes('Nodes', queries.runningNodes), @@ -60,7 +61,7 @@ local variables = import './variables.libsonnet'; + row.withPanels([ panels.timeSeries.bytes('Transport Rate', [queries.transportTXRate, queries.transportRXRate]), ]), - ], panelWidth=4, panelHeight=3), + ]), ), }, } diff --git a/elasticsearch-mixin/dashboards/util.libsonnet b/elasticsearch-mixin/dashboards/util.libsonnet new file mode 100644 index 00000000..388b06ae --- /dev/null +++ b/elasticsearch-mixin/dashboards/util.libsonnet @@ -0,0 +1,66 @@ +local g = import 'g.libsonnet'; +local panelUtil = g.util.panel; + +{ + local gridWidth = 24, + + // makeGrid returns an array of panels organized into a grid layout. + // This is a modified version of the grafonnet makeGrid function to + // calculate the width of each panel based on the number of panels. + makeGrid(panels, panelHeight=4, startY=0): + local sanitizePanels(ps) = + // Figure out the number of panels and the width of each panel + local numPanels = std.length(ps); + local panelWidth = std.floor(gridWidth / numPanels); + + // Sanitize the panels, this ensures tht the panels have the valid gridPos + std.map( + function(p) + local sanePanel = panelUtil.sanitizePanel(p, defaultHeight=panelHeight); + ( + if p.type == 'row' + then sanePanel { + panels: sanitizePanels(sanePanel.panels), + } + else sanePanel { + gridPos+: { + w: panelWidth, + }, + } + ), + ps + ); + + local sanitizedPanels = sanitizePanels(panels); + + local grouped = panelUtil.groupPanelsInRows(sanitizedPanels); + + local panelsBeforeRows = panelUtil.getPanelsBeforeNextRow(grouped); + local rowPanels = + std.filter( + function(p) p.type == 'row', + grouped + ); + + + local CalculateXforPanel(index, panel) = + local panelsPerRow = std.floor(gridWidth / panel.gridPos.w); + local col = std.mod(index, panelsPerRow); + panel { gridPos+: { x: panel.gridPos.w * col } }; + + + local panelsBeforeRowsWithX = std.mapWithIndex(CalculateXforPanel, panelsBeforeRows); + + local rowPanelsWithX = + std.map( + function(row) + row { panels: std.mapWithIndex(CalculateXforPanel, row.panels) }, + rowPanels + ); + + local uncollapsed = panelUtil.resolveCollapsedFlagOnRows(panelsBeforeRowsWithX + rowPanelsWithX); + + local normalized = panelUtil.normalizeY(uncollapsed); + + std.map(function(p) p { gridPos+: { y+: startY } }, normalized), +} From 7b340d1f736de9c07a989ebb70795045d9bb9235 Mon Sep 17 00:00:00 2001 From: Joe Adams Date: Sat, 7 Sep 2024 11:39:31 -0400 Subject: [PATCH 12/12] Update README and remove old dashboard Signed-off-by: Joe Adams --- elasticsearch-mixin/README.md | 16 +- examples/grafana/dashboard.json | 2250 ------------------------------- 2 files changed, 4 insertions(+), 2262 deletions(-) delete mode 100644 examples/grafana/dashboard.json diff --git a/elasticsearch-mixin/README.md b/elasticsearch-mixin/README.md index ef290faa..4f41eb2d 100644 --- a/elasticsearch-mixin/README.md +++ b/elasticsearch-mixin/README.md @@ -4,16 +4,6 @@ This is a mixin for the elasticsearch_exporter to define dashboards, alerts, and Good example of upstream mixin for reference: https://github.com/kubernetes-monitoring/kubernetes-mixin - -docker-compose -- docker-compose exec elasticsearch bash - - bin/elasticsearch-reset-password -u elastic -f -- login to grafana -- add prometheus datasource (http://prometheus:9090) -- http://127.0.0.1:3000 -- http://127.0.0.1:9090/targets?search= -- http://127.0.0.1:9114/metrics - ## Development ### JSONNET @@ -33,5 +23,7 @@ Grafana libraries for jsonnet: https://grafana.github.io/grafonnet/ ```jb install github.com/grafana/grafonnet/gen/grafonnet-latest@main``` -validate -go install github.com/grafana/dashboard-linter@latest +### Run the build +```bash +./scripts/compile-mixin.sh +``` diff --git a/examples/grafana/dashboard.json b/examples/grafana/dashboard.json deleted file mode 100644 index 6b1de5a8..00000000 --- a/examples/grafana/dashboard.json +++ /dev/null @@ -1,2250 +0,0 @@ -{ - "__inputs": [], - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "5.4.0" - }, - { - "type": "panel", - "id": "graph", - "name": "Graph", - "version": "5.0.0" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "5.0.0" - }, - { - "type": "panel", - "id": "singlestat", - "name": "Singlestat", - "version": "5.0.0" - } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 1, - "id": null, - "iteration": 1549021227642, - "links": [], - "panels": [ - { - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 90, - "title": "Cluster", - "type": "row" - }, - { - "cacheTimeout": null, - "colorBackground": true, - "colorPostfix": false, - "colorPrefix": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 12, - "x": 0, - "y": 1 - }, - "id": 92, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "Value", - "targets": [ - { - "expr": "scalar(elasticsearch_cluster_health_status{color=\"green\",cluster=~\"$cluster\"}) + scalar(elasticsearch_cluster_health_status{color=\"yellow\",cluster=~\"$cluster\"}) * 2 + scalar(elasticsearch_cluster_health_status{color=\"red\",cluster=~\"$cluster\"}) * 3", - "format": "time_series", - "instant": false, - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": "2,3", - "title": "Cluster Status", - "type": "singlestat", - "valueFontSize": "120%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - }, - { - "op": "=", - "text": "green", - "value": "1" - }, - { - "op": "=", - "text": "yellow", - "value": "2" - }, - { - "op": "=", - "text": "red", - "value": "3" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "datasource": "$server", - "editable": true, - "error": false, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 12, - "y": 1 - }, - "id": 8, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(elasticsearch_cluster_health_number_of_nodes{cluster=~\"$cluster\"})/count(elasticsearch_cluster_health_number_of_nodes{cluster=~\"$cluster\"})", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "", - "metric": "elasticsearch_cluster_health_number_of_nodes", - "refId": "A", - "step": 1800 - } - ], - "thresholds": "", - "timeFrom": null, - "timeShift": null, - "title": "Running Nodes", - "type": "singlestat", - "valueFontSize": "120%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 16, - "y": 1 - }, - "id": 94, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "elasticsearch_cluster_health_number_of_data_nodes{cluster=~\"$cluster\"}", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "", - "title": "Active Data Nodes", - "type": "singlestat", - "valueFontSize": "120%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": true, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 20, - "y": 1 - }, - "id": 96, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "elasticsearch_cluster_health_number_of_pending_tasks{cluster=~\"$cluster\"}", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "0.5,1", - "title": "Pending Tasks", - "type": "singlestat", - "valueFontSize": "120%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 4 - }, - "id": 76, - "panels": [], - "title": "Shards", - "type": "row" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 0, - "y": 5 - }, - "id": 78, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "elasticsearch_cluster_health_active_shards{cluster=~\"$cluster\"}", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "", - "title": "Active Shards", - "transparent": false, - "type": "singlestat", - "valueFontSize": "120%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 4, - "y": 5 - }, - "id": 80, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "elasticsearch_cluster_health_active_primary_shards{cluster=~\"$cluster\"}", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "", - "title": "Active Primary Shards", - "type": "singlestat", - "valueFontSize": "120%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": true, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 8, - "y": 5 - }, - "id": 82, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "elasticsearch_cluster_health_initializing_shards{cluster=~\"$cluster\"}", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "0.5,1", - "title": "Initializing Shards", - "type": "singlestat", - "valueFontSize": "120%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": true, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 12, - "y": 5 - }, - "id": 84, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "elasticsearch_cluster_health_relocating_shards{cluster=~\"$cluster\"}", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "0.5,1", - "title": "Relocating Shards", - "type": "singlestat", - "valueFontSize": "120%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": true, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 16, - "y": 5 - }, - "id": 86, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "elasticsearch_cluster_health_unassigned_shards{cluster=~\"$cluster\"}", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "0.5,1", - "title": "Unassigned Shards", - "type": "singlestat", - "valueFontSize": "120%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": true, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 20, - "y": 5 - }, - "id": 88, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "elasticsearch_cluster_health_delayed_unassigned_shards{cluster=~\"$cluster\"}", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "0.5,1", - "title": "Delayed Unassigned Shards", - "type": "singlestat", - "valueFontSize": "120%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 8 - }, - "id": 70, - "panels": [], - "title": "Documents", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$server", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 9 - }, - "id": 3, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(elasticsearch_indices_docs{cluster=~\"$cluster\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Documents", - "refId": "A", - "step": 240 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Documents indexed", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$server", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 9 - }, - "id": 4, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(elasticsearch_indices_store_size_bytes{cluster=~\"$cluster\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Index Size", - "refId": "A", - "step": 240 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Index Size", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "fill": 1, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 18 - }, - "id": 72, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(elasticsearch_indices_indexing_index_total{cluster=~\"$cluster\"}[1h])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{name}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Documents Indexed Rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "Documents/s", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "fill": 1, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 18 - }, - "id": 74, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(elasticsearch_indices_search_fetch_total{cluster=~\"$cluster\"}[1h])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{name}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Query Rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "Queris/s", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$server", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 27 - }, - "height": "", - "id": 64, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(elasticsearch_thread_pool_queue_count{cluster=~\"$cluster\", type!=\"management\"}) by (type)", - "interval": "", - "intervalFactor": 2, - "legendFormat": "Type: {{type}}", - "refId": "A", - "step": 240 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Queue Count", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 35 - }, - "id": 68, - "panels": [], - "title": "System", - "type": "row" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "datasource": "$server", - "editable": true, - "error": false, - "format": "bytes", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 8, - "x": 0, - "y": 36 - }, - "height": "", - "id": 12, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(node_memory_MemTotal_bytes{cluster=~\"$cluster\"})", - "intervalFactor": 2, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1800 - } - ], - "thresholds": "", - "title": "Total Memory", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "datasource": "$server", - "editable": true, - "error": false, - "format": "bytes", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 8, - "x": 8, - "y": 36 - }, - "height": "", - "id": 13, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(node_memory_MemFree_bytes{cluster=~\"$cluster\"})", - "intervalFactor": 2, - "legendFormat": "", - "refId": "A", - "step": 1800 - } - ], - "thresholds": "", - "title": "Total Memory Free", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "datasource": "$server", - "editable": true, - "error": false, - "format": "bytes", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 8, - "x": 16, - "y": 36 - }, - "height": "", - "id": 14, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(node_memory_MemAvailable_bytes{cluster=~\"$cluster\"})", - "intervalFactor": 2, - "legendFormat": "", - "refId": "A", - "step": 1800 - } - ], - "thresholds": "", - "title": "Total Memory Available", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$server", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 39 - }, - "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(elasticsearch_thread_pool_active_count{cluster=~\"$cluster\", type!=\"management\"}) by (type)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "Type: {{ type }}", - "refId": "A", - "step": 240 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Thread Pools", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$server", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 39 - }, - "id": 66, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(elasticsearch_thread_pool_rejected_count{cluster=~\"$cluster\", type!=\"management\"}[5m])", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{ name }} {{ type }}", - "refId": "A", - "step": 240 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Thread pool rejections", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$server", - "fill": 1, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 47 - }, - "id": 57, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "bucketAggs": [ - { - "id": "2", - "settings": { - "interval": "auto" - }, - "type": "date_histogram" - } - ], - "dsType": "elasticsearch", - "expr": "avg(irate(node_cpu_seconds_total{cluster=~\"$cluster\"}[60s])) by(mode) *100", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{ mode }}", - "metric": "elasticsearch_breakers_tripped", - "metrics": [ - { - "id": "1", - "type": "count" - } - ], - "refId": "A", - "step": 240, - "timeField": "failure_tstamp" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Avg. CPU Usage", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "", - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$server", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 47 - }, - "id": 28, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "rightSide": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "avg_over_time(elasticsearch_jvm_memory_used_bytes{area=\"heap\",cluster=~\"$cluster\"}[15m]) / elasticsearch_jvm_memory_max_bytes{area=\"heap\",cluster=~\"$cluster\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{ name }}", - "refId": "A", - "step": 240 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Avg Heap in 15min", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "label": "", - "logBase": 1, - "max": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$server", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 56 - }, - "id": 5, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(elasticsearch_transport_rx_packets_total{cluster=~\"$cluster\"}[5m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "RX", - "refId": "A", - "step": 240 - }, - { - "expr": "sum(rate(elasticsearch_transport_tx_packets_total{cluster=~\"$cluster\"}[5m])) * -1", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "TX", - "refId": "B", - "step": 240 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "RX/TX Rate 5m", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$server", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 56 - }, - "id": 65, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "irate(elasticsearch_jvm_gc_collection_seconds_sum{cluster=~\"$cluster\"}[1m])", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{ name }} {{ gc }}", - "refId": "A", - "step": 240 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "GC seconds", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - ], - "refresh": "1m", - "schemaVersion": 16, - "style": "dark", - "tags": [ - "infra", - "b2c", - "elastic" - ], - "templating": { - "list": [ - { - "current": { - "text": "Prometheus", - "value": "Prometheus" - }, - "hide": 0, - "label": "Server", - "name": "server", - "options": [], - "query": "prometheus", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" - }, - { - "allValue": null, - "current": {}, - "datasource": "$server", - "definition": "", - "hide": 0, - "includeAll": true, - "label": null, - "multi": true, - "name": "cluster", - "options": [], - "query": "label_values(elasticsearch_cluster_health_status,cluster)", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-12h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "utc", - "title": "Elasticsearch", - "uid": "n_nxrE_mk", - "version": 2 -}