Optionally suppress warnings about unreferenced NULL values (#46660)

#### Description
This enhancement to NULL value handling allows for cleaner logs when a
query cannot be formulated to hide NULL values. For example, the SQL
interface to pgBouncer is a handy way to monitor that service, but the
minimal SQL parsing of the interface does not allow for a subset of
columns to be returned. As a result, NULL values are common in the
returned rows, even if they will not be used for monitoring.

#### Link to tracking issue
Enhances #43985

#### Testing
Unit testing showing that the existence of ignore_null_values=true
suppresses warnings, unless the null values are in a referenced column.
Likewise, tests that show the default value, or explicitly setting the
value to false, keeps the warnings around. Errors always remain when a
columned holding NULL values is referenced.

#### Documentation
Documented the new option, and referenced it when talking about NULL
values in README.md.

Author: benchub

.chloggen/sqlqueryreceiver-ignore-null-values.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. receiver/filelog)
+component: receiver/sqlquery
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Add `ignore_null_values` config option to suppress NULL value warning logs
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [43985]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [user]

internal/sqlquery/config.go

@@ -88,6 +88,7 @@ type Query struct {
 	Logs               []LogsCfg   `mapstructure:"logs"`
 	TrackingColumn     string      `mapstructure:"tracking_column"`
 	TrackingStartValue string      `mapstructure:"tracking_start_value"`
+	IgnoreNullValues   bool        `mapstructure:"ignore_null_values"`
 }
 
 func (q Query) Validate() error {

internal/sqlquery/config.schema.yaml

@@ -84,6 +84,8 @@ $defs:
   query:
     type: object
     properties:
+      ignore_null_values:
+        type: boolean
       logs:
         type: array
         items:

internal/sqlquery/scraper.go

@@ -79,7 +79,9 @@ func (s *Scraper) ScrapeMetrics(ctx context.Context) (pmetric.Metrics, error) {
 		if !errors.Is(err, ErrNullValueWarning) {
 			return out, fmt.Errorf("scraper: %w", err)
 		}
-		s.Logger.Warn("problems encountered getting metric rows", zap.Error(err))
+		if !s.Query.IgnoreNullValues {
+			s.Logger.Warn("problems encountered getting metric rows", zap.Error(err))
+		}
 	}
 	ts := pcommon.NewTimestampFromTime(time.Now())
 	rms := out.ResourceMetrics()

internal/sqlquery/scraper_test.go

@@ -16,6 +16,7 @@ import (
 	"go.opentelemetry.io/collector/pdata/pmetric"
 	"go.opentelemetry.io/collector/scraper/scrapererror"
 	"go.uber.org/zap"
+	"go.uber.org/zap/zaptest/observer"
 )
 
 func TestScraper_ErrorOnStart(t *testing.T) {
@@ -362,6 +363,85 @@ func TestScraper_FakeDB_Warnings(t *testing.T) {
 	require.NoError(t, err)
 }
 
+func TestScraper_FakeDB_UnreferencedNullColumnWarning(t *testing.T) {
+	// An unreferenced NULL column (col_1) should only produce a warning when
+	// IgnoreNullValues is false (or unset). When true, no warning is logged.
+	tests := []struct {
+		name             string
+		ignoreNullValues bool
+		expectWarning    bool
+	}{
+		{name: "default", ignoreNullValues: false, expectWarning: true},
+		{name: "explicit_false", ignoreNullValues: false, expectWarning: true},
+		{name: "true", ignoreNullValues: true, expectWarning: false},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			db := fakeDB{rowVals: [][]any{{42, nil}}}
+			core, recorded := observer.New(zap.WarnLevel)
+			logger := zap.New(core)
+			scrpr := Scraper{
+				InstrumentationScope: pcommon.NewInstrumentationScope(),
+				Client:               NewDbClient(db, "", logger, TelemetryConfig{}),
+				Logger:               logger,
+				Query: Query{
+					IgnoreNullValues: tt.ignoreNullValues,
+					Metrics: []MetricCfg{{
+						MetricName:  "my.name",
+						ValueColumn: "col_0",
+						Description: "my description",
+						Unit:        "my-unit",
+					}},
+				},
+			}
+			_, err := scrpr.ScrapeMetrics(t.Context())
+			require.NoError(t, err)
+			if tt.expectWarning {
+				all := recorded.All()
+				require.Len(t, all, 1)
+				assert.Equal(t, "problems encountered getting metric rows", all[0].Message)
+			} else {
+				assert.Empty(t, recorded.All(), "expected no warnings when IgnoreNullValues is true")
+			}
+		})
+	}
+}
+
+func TestScraper_FakeDB_Warnings_ReferencedNullColumnStillErrors(t *testing.T) {
+	// A PartialScrapeError must be returned when a referenced column
+	// (ValueColumn) is NULL, regardless of the IgnoreNullValues setting.
+	tests := []struct {
+		name             string
+		ignoreNullValues bool
+	}{
+		{name: "default", ignoreNullValues: false},
+		{name: "explicit_false", ignoreNullValues: false},
+		{name: "true", ignoreNullValues: true},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			db := fakeDB{rowVals: [][]any{{nil, "ok"}}}
+			scrpr := Scraper{
+				InstrumentationScope: pcommon.NewInstrumentationScope(),
+				Client:               NewDbClient(db, "", zap.NewNop(), TelemetryConfig{}),
+				Logger:               zap.NewNop(),
+				Query: Query{
+					IgnoreNullValues: tt.ignoreNullValues,
+					Metrics: []MetricCfg{{
+						MetricName:       "my.name",
+						ValueColumn:      "col_0",
+						AttributeColumns: []string{"col_1"},
+					}},
+				},
+			}
+			_, err := scrpr.ScrapeMetrics(t.Context())
+			require.Error(t, err)
+			assert.True(t, scrapererror.IsPartialScrapeError(err))
+			assert.ErrorContains(t, err, "value_column 'col_0' not found in result set")
+		})
+	}
+}
+
 func TestScraper_FakeDB_MultiRows_Warnings(t *testing.T) {
 	db := fakeDB{rowVals: [][]any{{42, nil}, {43, nil}}}
 	logger := zap.NewNop()

receiver/sqlqueryreceiver/README.md

@@ -82,6 +82,7 @@ Additionally, each `query` section supports the following properties:
   See the below section [Tracking processed results](#tracking-processed-results).
 - `tracking_start_value` (optional, default `""`) Applies only to logs. In case of a parameterized query, defines the initial value for the parameter.
   See the below section [Tracking processed results](#tracking-processed-results).
+- `ignore_null_values` (optional, default `false`) When set to `true`, suppresses warning logs about NULL values encountered in query result columns. This is useful when queries return NULL in columns that are not referenced in the metric or log configuration.
 - `attribute_columns`(optional): a list of column names in the returned dataset used to set attributes on the signal.
   These attributes may be case-sensitive, depending on the driver (e.g. Oracle DB).
 
@@ -314,8 +315,8 @@ This produces three separate metrics (`pgbouncer.lists.pools`, `pgbouncer.lists.
 
 #### NULL values
 
-Avoid queries that produce any NULL values. If a query produces a NULL value, a warning will be logged. Furthermore,
-if a configuration references the column that produces a NULL value, an additional error will be logged. However, in
+If a query produces a NULL value, a warning will be logged unless `ignore_null_values` is set to true. Furthermore,
+if a configuration references the column that produces a NULL value, an error will always be logged. However, in
 either case, the receiver will continue to operate.
 
 #### Oracle DB Driver Example

receiver/sqlqueryreceiver/logs_receiver.go

@@ -304,7 +304,9 @@ func (queryReceiver *logsQueryReceiver) collect(ctx context.Context) (plog.Logs,
 		if !errors.Is(err, sqlquery.ErrNullValueWarning) {
 			return logs, fmt.Errorf("scraper: %w", err)
 		}
-		queryReceiver.logger.Warn("problems encountered getting log rows", zap.Error(err))
+		if !queryReceiver.query.IgnoreNullValues {
+			queryReceiver.logger.Warn("problems encountered getting log rows", zap.Error(err))
+		}
 	}
 
 	var errs []error

receiver/sqlqueryreceiver/logs_receiver_test.go

@@ -120,44 +120,101 @@ func TestLogsQueryReceiver_BothDatasourceFields(t *testing.T) {
 	require.NoError(t, receiver.Shutdown(ctx))
 }
 
-func TestLogsQueryReceiver_NullValue(t *testing.T) {
-	col1 := "col1"
-	col1Value := "42"
-	fakeClient := &sqlquery.FakeDBClient{
-		StringMaps: [][]sqlquery.StringMap{
-			{{col1: col1Value}},
-		},
-		// fakeClient.QueryRows will return ErrNullValueWarning on top of the StringMaps
-		ErrNullValueWarning: true,
+func TestLogsQueryReceiver_UnreferencedNullColumnWarning(t *testing.T) {
+	// An unreferenced NULL column should only produce a warning when
+	// IgnoreNullValues is false (or unset). When true, no warning is logged.
+	// In all cases the log record itself is collected successfully.
+	tests := []struct {
+		name             string
+		ignoreNullValues bool
+		expectWarning    bool
+	}{
+		{name: "default", ignoreNullValues: false, expectWarning: true},
+		{name: "explicit_false", ignoreNullValues: false, expectWarning: true},
+		{name: "true", ignoreNullValues: true, expectWarning: false},
 	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			col1 := "col1"
+			col1Value := "42"
+			fakeClient := &sqlquery.FakeDBClient{
+				StringMaps: [][]sqlquery.StringMap{
+					{{col1: col1Value}},
+				},
+				// fakeClient.QueryRows will return ErrNullValueWarning on top of the StringMaps
+				ErrNullValueWarning: true,
+			}
 
-	core, recorded := observer.New(zap.InfoLevel)
-	logger := zap.New(core)
+			core, recorded := observer.New(zap.WarnLevel)
+			logger := zap.New(core)
 
-	queryReceiver := logsQueryReceiver{
-		client: fakeClient,
-		query: sqlquery.Query{
-			Logs: []sqlquery.LogsCfg{
-				{
-					BodyColumn:       col1,
-					AttributeColumns: []string{col1},
+			queryReceiver := logsQueryReceiver{
+				client: fakeClient,
+				query: sqlquery.Query{
+					IgnoreNullValues: tt.ignoreNullValues,
+					Logs: []sqlquery.LogsCfg{
+						{
+							BodyColumn:       col1,
+							AttributeColumns: []string{col1},
+						},
+					},
 				},
-			},
-		},
-		logger: logger,
+				logger: logger,
+			}
+			logs, err := queryReceiver.collect(t.Context())
+			assert.NoError(t, err)
+			assert.Equal(t, 1, logs.LogRecordCount())
+			assert.Equal(t, col1Value, logs.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Body().Str())
+
+			if tt.expectWarning {
+				all := recorded.All()
+				require.Len(t, all, 1)
+				assert.Equal(t, "problems encountered getting log rows", all[0].Message)
+				assert.Equal(t, sqlquery.ErrNullValueWarning.Error(), all[0].ContextMap()["error"])
+			} else {
+				assert.Empty(t, recorded.All(), "expected no warnings when IgnoreNullValues is true")
+			}
+		})
+	}
+}
+
+func TestLogsQueryReceiver_NullValue_ReferencedNullColumnStillErrors(t *testing.T) {
+	// An error must be returned when a referenced column (BodyColumn) is NULL,
+	// regardless of the IgnoreNullValues setting.
+	tests := []struct {
+		name             string
+		ignoreNullValues bool
+	}{
+		{name: "default", ignoreNullValues: false},
+		{name: "explicit_false", ignoreNullValues: false},
+		{name: "true", ignoreNullValues: true},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			fakeClient := &sqlquery.FakeDBClient{
+				StringMaps: [][]sqlquery.StringMap{
+					{{"other_col": "val"}}, // BodyColumn "missing_col" is absent
+				},
+				ErrNullValueWarning: true,
+			}
+
+			queryReceiver := logsQueryReceiver{
+				client: fakeClient,
+				query: sqlquery.Query{
+					IgnoreNullValues: tt.ignoreNullValues,
+					Logs: []sqlquery.LogsCfg{
+						{
+							BodyColumn: "missing_col",
+						},
+					},
+				},
+				logger: zap.NewNop(),
+			}
+			_, err := queryReceiver.collect(t.Context())
+			require.Error(t, err)
+			assert.ErrorContains(t, err, "body_column 'missing_col' not found in result set")
+		})
 	}
-	// ensure that the logs are collected successfully
-	logs, err := queryReceiver.collect(t.Context())
-	assert.NoError(t, err)
-	assert.Equal(t, 1, logs.LogRecordCount())
-	assert.Equal(t, col1Value, logs.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Body().Str())
-
-	// ensure that the warning is logged
-	all := recorded.All()
-	require.Len(t, all, 1)
-	entry := all[0]
-	require.Equal(t, "problems encountered getting log rows", entry.Message)
-	require.Equal(t, sqlquery.ErrNullValueWarning.Error(), entry.ContextMap()["error"])
 }
 
 func TestLogsReceiver_InitialDelay(t *testing.T) {