running metric collection with read only connection.

The read only connection is set up with follower reads, to minimize contention.

Author: sravotto

.gitignore

@@ -13,3 +13,9 @@
 
 # Dependency directories (remove the comment below to include it)
 # vendor/
+scripts/
+tmp/
+*.sh
+certs
+visus
+visus-*

README.md

@@ -1,4 +1,5 @@
 # visus
+
 Visus (latin for "the action of looking") enables users to collect metrics using arbitrary SQL queries and expose them in a Prometheus format. Optionally, it can be configured to filter the metrics CockroachDB available in the `/_status/vars` endpoint.
 
 Visus runs as a sidecar on each node of a CockroachDB cluster, as shown in the diagram below.
@@ -29,11 +30,12 @@ LIMIT
 
 The Prometheus collectors will add additional labels to track the cluster name as well as the instance where the metrics are coming from.
 
-## Database Security 
+## Database Security
+
 It is recommended to use separate users for managing the configuration and to run the sidecar.
 The sidecar needs `SELECT ON TABLES` privileges on the `_visus` database to read the configuration. It also needs the `SELECT,INSERT,UPDATE` privileges on the `_visus.node` table, used to determine which node should collect cluster wide metrics.  
 
-To run many of the sample collections available in the examples directory, 
+To run many of the sample collections available in the examples directory,
 the 'VIEWACTIVITY' option should be granted to the user.
 The `./visus init` command will provision a `visus` user with the minimal privileges to run the sidecar. Defining new collection may require additional privileges, depending on what data the SQL query associated to the collection has to access.
 
@@ -172,8 +174,9 @@ $VISUS_USER start --bind-addr :8888 --insecure --endpoint "/_status/custom"
 
 ## Histogram rewriting
 
-Visus can also act as a proxy to filter and rewrite CockroachDB histograms (v22.1 and earlier) from a log-2 linear format (HDR histograms) to a log-10 linear format. 
+Visus can also act as a proxy to filter and rewrite CockroachDB histograms (v22.1 and earlier) from a log-2 linear format (HDR histograms) to a log-10 linear format.
 Users can specify which histograms to rewrite based on a regular expression. For instance to rewrite all the histograms that match "^sql_exec_latency$" and keep buckets between 1ms and 20sec, we specify in the configuration file `latency.yaml`:
+
 ```yaml
 name: latency
 regex: ^sql_exec_latency$

examples/cluster_sqlactivity.yaml

@@ -0,0 +1,89 @@
+name: cluster_sqlactivity
+enabled: true
+scope: node
+frequency: 20
+maxresults: 20
+labels: [statement,application,database]
+metrics:
+  - name : exec_count
+    kind : counter
+    help : number of times the statement was executed.
+  - name : net_bytes
+    kind : gauge
+    help : amount of data transferred over the network.
+  - name : max_disk
+    kind : gauge
+    help : maximum data transferred from the disk.
+  - name : run_lat
+    kind : gauge
+    help : average runtime.
+  - name : rows_read
+    kind : gauge
+    help : average number of rows read from disk. 
+  - name : rows_avg
+    kind : gauge
+    help : average number of rows returned.  
+  - name : bytes_avg
+    kind : gauge
+    help : average number of bytes read from disk.  
+  - name : total_lat
+    kind : gauge
+    help : total runtime for the statement.      
+  - name : max_retries
+    kind : gauge
+    help : cumulative number of automatic retries.      
+  - name : max_mem
+    kind : gauge
+    help : maximum memory used by the statement.          
+  - name : cont_time
+    kind : gauge
+    help : number of times the statement was executed.            
+query: 
+    WITH 
+    max AS (select max(aggregated_ts) from crdb_internal.statement_statistics),
+    total AS (
+    SELECT
+      encode(fingerprint_id, 'hex') as statement,
+      app_name as application,
+      metadata->'db' as database,
+      sum((statistics->'execution_statistics'->'cnt')::float8) AS exec_count,
+      max((statistics->'execution_statistics'->'maxDiskUsage'->'mean')::float8) AS max_disk,
+      sum((statistics->'execution_statistics'->'networkBytes'->'mean')::float8  * (statistics->'execution_statistics'->'cnt')::float8 )  AS net_bytes,
+      sum((statistics->'statistics'->'runLat'->'mean')::float8 * (statistics->'execution_statistics'->'cnt')::float8 ) AS run_lat,
+      sum((statistics->'statistics'->'rowsRead'->'mean')::float8 * (statistics->'execution_statistics'->'cnt')::float8 )  AS rows_read,
+      sum((statistics->'statistics'->'numRows'->'mean')::float8  * (statistics->'execution_statistics'->'cnt')::float8)  AS rows_num,
+      sum((statistics->'statistics'->'bytesRead'->'mean')::float8  * (statistics->'execution_statistics'->'cnt')::float8) AS bytes_read,
+      sum((statistics->'statistics'->'runLat'->'mean')::float8 *  (statistics->'execution_statistics'->'cnt')::float8) AS total_lat,
+      sum((statistics->'statistics'->'maxRetries'->'mean')::float8) AS max_retries,
+      max((statistics->'execution_statistics'->'maxMemUsage'->'mean')::float8) AS max_mem,
+      sum((statistics->'execution_statistics'->'contentionTime'->'mean')::float8  *  (statistics->'execution_statistics'->'cnt')::float8) AS cont_time
+    FROM
+      crdb_internal.statement_statistics
+    WHERE
+      app_name NOT LIKE '$ internal-%'
+      AND aggregated_ts = (select * from max)
+      AND aggregation_interval = '1h'  
+    GROUP BY
+      statement, application, database  
+  )
+  select 
+    statement, application, database,
+    exec_count,
+    net_bytes / exec_count as net_bytes,
+    max_disk,
+    run_lat / exec_count as  run_lat,
+    rows_read / exec_count as rows_read,
+    rows_num / exec_count as rows_avg,
+    bytes_read / exec_count as bytes_avg,
+    total_lat,
+    max_retries,
+    max_mem,
+    cont_time / exec_count as cont_time
+    FROM 
+    total  
+    WHERE exec_count > 0
+    
+    ORDER BY
+      total_lat DESC
+    LIMIT
+      $1;

examples/sqlactivity.yaml

@@ -1,8 +1,8 @@
 name: sqlactivity
 enabled: true
 scope: node
-frequency: 10
-maxresults: 10
+frequency: 5
+maxresults: 20
 labels: [statement,application,database]
 metrics:
   - name : exec_count

go.mod

@@ -4,55 +4,55 @@ go 1.19
 
 require (
 	github.com/NYTimes/gziphandler v1.1.1
-	github.com/cockroachdb/crlfmt v0.0.0-20220610162206-024b567ce87b
-	github.com/creasty/defaults v1.6.0
-	github.com/go-co-op/gocron v1.32.1
+	github.com/cockroachdb/crlfmt v0.0.0-20230505164321-461e8663b4b4
+	github.com/creasty/defaults v1.7.0
+	github.com/go-co-op/gocron v1.35.3
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da
-	github.com/jackc/pgconn v1.14.0
+	github.com/jackc/pgconn v1.14.1
 	github.com/jackc/pgtype v1.14.0
 	github.com/jackc/pgx/v4 v4.18.1
-	github.com/joonix/log v0.0.0-20200409080653-9c1d2ceb5f1d
+	github.com/joonix/log v0.0.0-20230221083239-7988383bab32
 	github.com/pashagolub/pgxmock v1.8.0
 	github.com/pkg/errors v0.9.1
-	github.com/prometheus/client_golang v1.15.1
-	github.com/prometheus/client_model v0.4.0
-	github.com/prometheus/common v0.44.0
-	github.com/sirupsen/logrus v1.9.0
-	github.com/spf13/cobra v1.6.1
+	github.com/prometheus/client_golang v1.17.0
+	github.com/prometheus/client_model v0.5.0
+	github.com/prometheus/common v0.45.0
+	github.com/sirupsen/logrus v1.9.3
+	github.com/spf13/cobra v1.8.0
 	github.com/stretchr/testify v1.8.4
-	golang.org/x/lint v0.0.0-20200302205851-738671d3881b
-	golang.org/x/tools v0.6.0
-	google.golang.org/protobuf v1.30.0
+	golang.org/x/lint v0.0.0-20210508222113-6edffad5e616
+	golang.org/x/tools v0.14.0
+	google.golang.org/protobuf v1.31.0
 	gopkg.in/yaml.v3 v3.0.1
-	honnef.co/go/tools v0.4.2
+	honnef.co/go/tools v0.4.6
 )
 
 require (
-	github.com/BurntSushi/toml v1.2.1 // indirect
+	github.com/BurntSushi/toml v1.3.2 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
-	github.com/cockroachdb/gostdlib v1.13.0 // indirect
-	github.com/cockroachdb/ttycolor v0.0.0-20180709150743-a1d5aaeb377d // indirect
+	github.com/cockroachdb/gostdlib v1.19.0 // indirect
+	github.com/cockroachdb/ttycolor v0.0.0-20210902133924-c7d7dcdde4e8 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
-	github.com/google/uuid v1.3.0 // indirect
-	github.com/inconshreveable/mousetrap v1.0.1 // indirect
+	github.com/google/uuid v1.4.0 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jackc/chunkreader/v2 v2.0.1 // indirect
 	github.com/jackc/pgio v1.0.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgproto3/v2 v2.3.2 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
 	github.com/jackc/puddle v1.3.0 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
+	github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/procfs v0.9.0 // indirect
+	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/robfig/cron/v3 v3.0.1 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	go.uber.org/atomic v1.9.0 // indirect
-	golang.org/x/crypto v0.6.0 // indirect
-	golang.org/x/exp/typeparams v0.0.0-20221208152030-732eee02a75a // indirect
-	golang.org/x/mod v0.8.0 // indirect
-	golang.org/x/sys v0.8.0 // indirect
-	golang.org/x/text v0.9.0 // indirect
-	google.golang.org/genproto v0.0.0-20200825200019-8632dd797987 // indirect
+	go.uber.org/atomic v1.11.0 // indirect
+	golang.org/x/crypto v0.14.0 // indirect
+	golang.org/x/exp/typeparams v0.0.0-20231006140011-7918f672742d // indirect
+	golang.org/x/mod v0.14.0 // indirect
+	golang.org/x/sys v0.14.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+	google.golang.org/genproto v0.0.0-20231030173426-d783a09b4405 // indirect
 )