Skip to content

Commit a920d40

Browse files
committed
Query performance monitoring
Query performance monitoring undo change remove test query query performance monitoring undo changelog changes Query performance monitoring Query performance monitoring slow query len check in individual queries metrics populate trim query text lint issue fix undo changelog file change test remove OH queries remove OH queries reusing existing DB connection (#22) reusing existing DB connection DB specific metric collection (#23) DB specific metric collection query response time threshold (#24) query response time threshold change in individual query (#25) change in individual query version specific queries (#26) verison specific queries fix database name (#27) fix database name test test min max threshold on limit (#28) min max threshold on limit Feat version specific individual query (#29) * version specific indivual query Feat version specific individual query (#30) Version specific indivual query fix query id , ingestData:false and reuse individual version specific query (#31) fix query id , ingestData:false and reuse individual version specific query database list check (#32) database list check publish only if there are metrics (#33) publish only if there are metrics undo change log change
1 parent c075367 commit a920d40

18 files changed

+1070
-4
lines changed

go.mod

+1
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ require (
66
github.com/blang/semver/v4 v4.0.0
77
github.com/jmoiron/sqlx v1.4.0
88
github.com/lib/pq v1.10.9
9+
github.com/mitchellh/mapstructure v1.5.0
910
github.com/newrelic/infra-integrations-sdk/v3 v3.9.1
1011
github.com/stretchr/testify v1.10.0
1112
github.com/xeipuuv/gojsonschema v1.2.0

go.sum

+2
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@ github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
1818
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
1919
github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
2020
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
21+
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
22+
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
2123
github.com/newrelic/infra-integrations-sdk/v3 v3.9.1 h1:dCtVLsYNHWTQ5aAlAaHroomOUlqxlGTrdi6XTlvBDfI=
2224
github.com/newrelic/infra-integrations-sdk/v3 v3.9.1/go.mod h1:yPeidhcq9Cla0QDquGXH0KqvS2k9xtetFOD7aLA0Z8M=
2325
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=

src/args/argument_list.go

+3
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,9 @@ type ArgumentList struct {
3131
CollectDbLockMetrics bool `default:"false" help:"If true, enables collection of lock metrics for the specified database. (Note: requires that the 'tablefunc' extension is installed)"` //nolint: stylecheck
3232
CollectBloatMetrics bool `default:"true" help:"Enable collecting bloat metrics which can be performance intensive"`
3333
ShowVersion bool `default:"false" help:"Print build information and exit"`
34+
EnableQueryMonitoring bool `default:"true" help:"Query monitoring is enabled by default. Set to false to disable."`
35+
QueryResponseTimeThreshold int `default:"0" help:"Threshold in milliseconds for query response time to fetch individual query performance metrics."`
36+
QueryCountThreshold int `default:"20" help:"Query count limit for fetching grouped slow and individual query performance metrics."`
3437
}
3538

3639
// Validate validates PostgreSQl arguments

src/connection/pgsql_connection.go

+1-2
Original file line numberDiff line numberDiff line change
@@ -149,7 +149,7 @@ func (p PGSQLConnection) HaveExtensionInSchema(extensionName, schemaName string)
149149
return true
150150
}
151151

152-
// createConnectionURL creates the connection string. A list of paramters
152+
// createConnectionURL creates the connection string. A list of parameters
153153
// can be found here https://godoc.org/github.com/lib/pq#hdr-Connection_String_Parameters
154154
func createConnectionURL(ci *connectionInfo, database string) string {
155155
connectionURL := &url.URL{
@@ -170,7 +170,6 @@ func createConnectionURL(ci *connectionInfo, database string) string {
170170
}
171171

172172
connectionURL.RawQuery = query.Encode()
173-
174173
return connectionURL.String()
175174
}
176175

src/main.go

+8-2
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
//go:generate goversioninfo
21
package main
32

43
import (
@@ -7,6 +6,8 @@ import (
76
"runtime"
87
"strings"
98

9+
queryperformancemonitoring "github.com/newrelic/nri-postgresql/src/query-performance-monitoring"
10+
1011
"github.com/newrelic/infra-integrations-sdk/v3/integration"
1112
"github.com/newrelic/infra-integrations-sdk/v3/log"
1213
"github.com/newrelic/nri-postgresql/src/args"
@@ -27,6 +28,7 @@ var (
2728
)
2829

2930
func main() {
31+
3032
var args args.ArgumentList
3133
// Create Integration
3234
pgIntegration, err := integration.New(integrationName, integrationVersion, integration.Args(&args))
@@ -62,7 +64,6 @@ func main() {
6264
log.Error("Error creating list of entities to collect: %s", err)
6365
os.Exit(1)
6466
}
65-
6667
instance, err := pgIntegration.Entity(fmt.Sprintf("%s:%s", args.Hostname, args.Port), "pg-instance")
6768
if err != nil {
6869
log.Error("Error creating instance entity: %s", err.Error())
@@ -89,4 +90,9 @@ func main() {
8990
if err = pgIntegration.Publish(); err != nil {
9091
log.Error(err.Error())
9192
}
93+
94+
if args.EnableQueryMonitoring {
95+
queryperformancemonitoring.QueryPerformanceMain(args, pgIntegration, collectionList)
96+
}
97+
9298
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
package commonutils
2+
3+
import (
4+
"fmt"
5+
"strings"
6+
7+
"github.com/newrelic/nri-postgresql/src/collection"
8+
)
9+
10+
func GetQuotedStringFromArray(array []string) string {
11+
var quotedDatabaseNames = make([]string, 0)
12+
for _, name := range array {
13+
quotedDatabaseNames = append(quotedDatabaseNames, fmt.Sprintf("'%s'", name))
14+
}
15+
return strings.Join(quotedDatabaseNames, ",")
16+
}
17+
18+
func GetDatabaseListInString(dbList collection.DatabaseList) string {
19+
var databaseNames = make([]string, 0)
20+
for dbName := range dbList {
21+
databaseNames = append(databaseNames, dbName)
22+
}
23+
if len(databaseNames) == 0 {
24+
return ""
25+
}
26+
return GetQuotedStringFromArray(databaseNames)
27+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
package commonutils
2+
3+
const MAX_QUERY_THRESHOLD = 30
4+
const MAX_INDIVIDUAL_QUERY_THRESHOLD = 10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
package commonutils
2+
3+
import (
4+
"crypto/rand"
5+
"fmt"
6+
_ "github.com/lib/pq"
7+
"github.com/newrelic/infra-integrations-sdk/v3/data/metric"
8+
"github.com/newrelic/infra-integrations-sdk/v3/integration"
9+
"github.com/newrelic/infra-integrations-sdk/v3/log"
10+
"github.com/newrelic/nri-postgresql/src/args"
11+
"math/big"
12+
"reflect"
13+
"time"
14+
)
15+
16+
const publishThreshold = 100
17+
const randomIntRange = 1000000
18+
19+
func SetMetric(metricSet *metric.Set, name string, value interface{}, sourceType string) {
20+
switch sourceType {
21+
case `gauge`:
22+
err := metricSet.SetMetric(name, value, metric.GAUGE)
23+
if err != nil {
24+
return
25+
}
26+
case `attribute`:
27+
err := metricSet.SetMetric(name, value, metric.ATTRIBUTE)
28+
if err != nil {
29+
return
30+
}
31+
default:
32+
err := metricSet.SetMetric(name, value, metric.GAUGE)
33+
if err != nil {
34+
return
35+
}
36+
}
37+
}
38+
39+
func IngestMetric(metricList []interface{}, eventName string, pgIntegration *integration.Integration, args args.ArgumentList) {
40+
instanceEntity, err := createEntity(pgIntegration, args)
41+
if err != nil {
42+
log.Error("Error creating entity: %v", err)
43+
return
44+
}
45+
46+
metricCount := 0
47+
lenOfMetricList := len(metricList)
48+
49+
for _, model := range metricList {
50+
if model == nil {
51+
continue
52+
}
53+
metricCount += 1
54+
metricSet := instanceEntity.NewMetricSet(eventName)
55+
56+
processModel(model, metricSet)
57+
58+
if metricCount == publishThreshold || metricCount == lenOfMetricList {
59+
metricCount = 0
60+
if err := publishMetrics(pgIntegration, &instanceEntity, args); err != nil {
61+
log.Error("Error publishing metrics: %v", err)
62+
return
63+
}
64+
}
65+
}
66+
if metricCount > 0 {
67+
if err := publishMetrics(pgIntegration, &instanceEntity, args); err != nil {
68+
log.Error("Error publishing metrics: %v", err)
69+
return
70+
}
71+
}
72+
}
73+
74+
func createEntity(pgIntegration *integration.Integration, args args.ArgumentList) (*integration.Entity, error) {
75+
return pgIntegration.Entity(fmt.Sprintf("%s:%s", args.Hostname, args.Port), "pg-instance")
76+
}
77+
78+
func processModel(model interface{}, metricSet *metric.Set) {
79+
modelValue := reflect.ValueOf(model)
80+
if modelValue.Kind() == reflect.Ptr {
81+
modelValue = modelValue.Elem()
82+
}
83+
if !modelValue.IsValid() || modelValue.Kind() != reflect.Struct {
84+
return
85+
}
86+
87+
modelType := reflect.TypeOf(model)
88+
89+
for i := 0; i < modelValue.NumField(); i++ {
90+
field := modelValue.Field(i)
91+
fieldType := modelType.Field(i)
92+
metricName := fieldType.Tag.Get("metric_name")
93+
sourceType := fieldType.Tag.Get("source_type")
94+
ingestData := fieldType.Tag.Get("ingest_data")
95+
96+
if ingestData == "false" {
97+
continue
98+
}
99+
100+
if field.Kind() == reflect.Ptr && !field.IsNil() {
101+
SetMetric(metricSet, metricName, field.Elem().Interface(), sourceType)
102+
} else if field.Kind() != reflect.Ptr {
103+
SetMetric(metricSet, metricName, field.Interface(), sourceType)
104+
}
105+
}
106+
}
107+
108+
func publishMetrics(pgIntegration *integration.Integration, instanceEntity **integration.Entity, args args.ArgumentList) error {
109+
if err := pgIntegration.Publish(); err != nil {
110+
return err
111+
}
112+
var err error
113+
*instanceEntity, err = pgIntegration.Entity(fmt.Sprintf("%s:%s", args.Hostname, args.Port), "pg-instance")
114+
return err
115+
}
116+
117+
func GenerateRandomIntegerString(queryID string) *string {
118+
randomInt, err := rand.Int(rand.Reader, big.NewInt(randomIntRange))
119+
if err != nil {
120+
return nil
121+
}
122+
currentTime := time.Now().Format("20060102150405")
123+
result := fmt.Sprintf("%s-%d-%s", queryID, randomInt.Int64(), currentTime)
124+
return &result
125+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
package commonutils
2+
3+
import (
4+
"fmt"
5+
"regexp"
6+
"strconv"
7+
8+
"github.com/newrelic/infra-integrations-sdk/v3/log"
9+
performancedbconnection "github.com/newrelic/nri-postgresql/src/connection"
10+
"github.com/newrelic/nri-postgresql/src/query-performance-monitoring/queries"
11+
)
12+
13+
func FetchVersion(conn *performancedbconnection.PGSQLConnection) (int, error) {
14+
var versionStr string
15+
rows, err := conn.Queryx("SELECT version()")
16+
if err != nil {
17+
log.Error("Error executing query: %v", err)
18+
return 0, err
19+
}
20+
defer rows.Close()
21+
22+
if rows.Next() {
23+
if err := rows.Scan(&versionStr); err != nil {
24+
log.Error("Error scanning version: %v", err)
25+
return 0, err
26+
}
27+
}
28+
re := regexp.MustCompile(`PostgreSQL (\d+)\.`)
29+
matches := re.FindStringSubmatch(versionStr)
30+
if len(matches) < 2 {
31+
log.Error("Unable to parse PostgreSQL version from string: %s", versionStr)
32+
return 0, fmt.Errorf("unable to parse PostgreSQL version from string: %s", versionStr)
33+
}
34+
35+
version, err := strconv.Atoi(matches[1])
36+
log.Debug("version", version)
37+
if err != nil {
38+
log.Error("Error converting version to integer: %v", err)
39+
return 0, err
40+
}
41+
return version, nil
42+
}
43+
44+
func FetchVersionSpecificSlowQueries(conn *performancedbconnection.PGSQLConnection) (string, error) {
45+
version, err := FetchVersion(conn)
46+
if err != nil {
47+
return "", err
48+
}
49+
switch {
50+
case version == 12:
51+
return queries.SlowQueriesForV12, nil
52+
case version >= 13:
53+
return queries.SlowQueriesForV13AndAbove, nil
54+
default:
55+
return "", fmt.Errorf("unsupported PostgreSQL version %d", version)
56+
}
57+
}
58+
59+
func FetchVersionSpecificBlockingQueries(conn *performancedbconnection.PGSQLConnection) (string, error) {
60+
version, err := FetchVersion(conn)
61+
if err != nil {
62+
return "", err
63+
}
64+
switch {
65+
case version == 12, version == 13:
66+
return queries.BlockingQueriesForV12AndV13, nil
67+
case version >= 14:
68+
return queries.BlockingQueriesForV14AndAbove, nil
69+
default:
70+
return "", fmt.Errorf("unsupported PostgreSQL version: %d", version)
71+
}
72+
}
73+
74+
func FetchVersionSpecificIndividualQueries(conn *performancedbconnection.PGSQLConnection) (string, error) {
75+
version, err := FetchVersion(conn)
76+
if err != nil {
77+
return "", err
78+
}
79+
switch {
80+
case version == 12:
81+
return queries.IndividualQuerySearchV12, nil
82+
case version >= 13:
83+
return queries.IndividualQuerySearchV13AndAbove, nil
84+
default:
85+
return "", fmt.Errorf("unsupported PostgreSQL version %d", version)
86+
}
87+
}

0 commit comments

Comments
 (0)