feat: add useMemAvailable feature gate

.chloggen/mem_used_feature_gate.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: hostmetricsreceiver
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Add useMemAvailable feature gate to use the MemAvailable kernel's statistic to compute the "used" memory usage
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [42221]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []

receiver/hostmetricsreceiver/internal/scraper/memoryscraper/memory_scraper_linux.go

@@ -7,13 +7,28 @@ package memoryscraper // import "github.com/open-telemetry/opentelemetry-collect
 
 import (
 	"github.com/shirou/gopsutil/v4/mem"
+	"go.opentelemetry.io/collector/featuregate"
 	"go.opentelemetry.io/collector/pdata/pcommon"
 
 	"github.com/open-telemetry/opentelemetry-collector-contrib/receiver/hostmetricsreceiver/internal/scraper/memoryscraper/internal/metadata"
 )
 
+var useMemAvailable = featuregate.GlobalRegistry().MustRegister(
+	"receiver.hostmetricsreceiver.UseLinuxMemAvailable",
+	featuregate.StageAlpha,
+	featuregate.WithRegisterFromVersion("v0.134.0"),
+	featuregate.WithRegisterDescription("When enabled, the used value for the system.memory.usage and system.memory.utilization metrics will be based on the Linux kernel’s MemAvailable statistic instead of MemFree, Buffers, and Cached."),
+	featuregate.WithRegisterReferenceURL("https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/42221"),
+)
+
 func (s *memoryScraper) recordMemoryUsageMetric(now pcommon.Timestamp, memInfo *mem.VirtualMemoryStat) {
-	s.mb.RecordSystemMemoryUsageDataPoint(now, int64(memInfo.Used), metadata.AttributeStateUsed)
+	// TODO: rely on memInfo.Used value once https://github.com/shirou/gopsutil/pull/1882 is released
+	if useMemAvailable.IsEnabled() {
+		s.mb.RecordSystemMemoryUsageDataPoint(now, int64(memInfo.Total-memInfo.Available), metadata.AttributeStateUsed)
+	} else {
+		// gopsutil legacy "Used" memory formula = Total - Free - Buffers - Cache
+		s.mb.RecordSystemMemoryUsageDataPoint(now, int64(memInfo.Total-memInfo.Free-memInfo.Buffers-memInfo.Cached), metadata.AttributeStateUsed)
+	}
 	s.mb.RecordSystemMemoryUsageDataPoint(now, int64(memInfo.Free), metadata.AttributeStateFree)
 	s.mb.RecordSystemMemoryUsageDataPoint(now, int64(memInfo.Buffers), metadata.AttributeStateBuffered)
 	s.mb.RecordSystemMemoryUsageDataPoint(now, int64(memInfo.Cached), metadata.AttributeStateCached)
@@ -22,7 +37,13 @@ func (s *memoryScraper) recordMemoryUsageMetric(now pcommon.Timestamp, memInfo *
 }
 
 func (s *memoryScraper) recordMemoryUtilizationMetric(now pcommon.Timestamp, memInfo *mem.VirtualMemoryStat) {
-	s.mb.RecordSystemMemoryUtilizationDataPoint(now, float64(memInfo.Used)/float64(memInfo.Total), metadata.AttributeStateUsed)
+	// TODO: rely on memInfo.Used value once https://github.com/shirou/gopsutil/pull/1882 is released
+	if useMemAvailable.IsEnabled() {
+		s.mb.RecordSystemMemoryUtilizationDataPoint(now, float64(memInfo.Total-memInfo.Available)/float64(memInfo.Total), metadata.AttributeStateUsed)
+	} else {
+		// gopsutil legacy "Used" memory formula = Total - Free - Buffers - Cache
+		s.mb.RecordSystemMemoryUtilizationDataPoint(now, float64(memInfo.Total-memInfo.Free-memInfo.Buffers-memInfo.Cached)/float64(memInfo.Total), metadata.AttributeStateUsed)
+	}
 	s.mb.RecordSystemMemoryUtilizationDataPoint(now, float64(memInfo.Free)/float64(memInfo.Total), metadata.AttributeStateFree)
 	s.mb.RecordSystemMemoryUtilizationDataPoint(now, float64(memInfo.Buffers)/float64(memInfo.Total), metadata.AttributeStateBuffered)
 	s.mb.RecordSystemMemoryUtilizationDataPoint(now, float64(memInfo.Cached)/float64(memInfo.Total), metadata.AttributeStateCached)

receiver/hostmetricsreceiver/internal/scraper/memoryscraper/memory_scraper_linux_test.go

@@ -0,0 +1,55 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package memoryscraper
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/collector/component/componenttest"
+	"go.opentelemetry.io/collector/featuregate"
+	"go.opentelemetry.io/collector/pdata/pcommon"
+	"go.opentelemetry.io/collector/scraper/scrapertest"
+
+	"github.com/open-telemetry/opentelemetry-collector-contrib/receiver/hostmetricsreceiver/internal/scraper/memoryscraper/internal/metadata"
+)
+
+func TestScrape_UseMemAvailable(t *testing.T) {
+	mbc := metadata.DefaultMetricsBuilderConfig()
+	mbc.Metrics.SystemMemoryUtilization.Enabled = true
+	mbc.Metrics.SystemMemoryUsage.Enabled = true
+	scraperConfig := Config{
+		MetricsBuilderConfig: mbc,
+	}
+	scraper := newMemoryScraper(t.Context(), scrapertest.NewNopSettings(metadata.Type), &scraperConfig)
+
+	err := scraper.start(t.Context(), componenttest.NewNopHost())
+	require.NoError(t, err, "Failed to initialize memory scraper: %v", err)
+
+	memInfo, err := scraper.virtualMemory(t.Context())
+	require.NoError(t, err)
+	require.NotNil(t, memInfo)
+
+	scraper.recordMemoryUsageMetric(pcommon.NewTimestampFromTime(time.Now()), memInfo)
+	scraper.recordMemoryUtilizationMetric(pcommon.NewTimestampFromTime(time.Now()), memInfo)
+	legacyMd := scraper.mb.Emit()
+
+	// enable feature gate
+	_ = featuregate.GlobalRegistry().Set(
+		"receiver.hostmetricsreceiver.UseLinuxMemAvailable", true)
+	t.Cleanup(func() {
+		_ = featuregate.GlobalRegistry().Set("receiver.hostmetricsreceiver.UseLinuxMemAvailable", false)
+	})
+	scraper.recordMemoryUsageMetric(pcommon.NewTimestampFromTime(time.Now()), memInfo)
+	scraper.recordMemoryUtilizationMetric(pcommon.NewTimestampFromTime(time.Now()), memInfo)
+	memUsedMd := scraper.mb.Emit()
+
+	// Used memory calculation based on MemAvailable is greater than "Total
+	// - Free - Buffers - Cache" as it takes into account the amount of
+	// Cached memory that is not freeable.
+	assert.Greater(t, memUsedMd.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(0).IntValue(), legacyMd.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(0).IntValue(), "system.memory.usage for the used state should be greater when computed using memAvailable")
+	assert.Greater(t, memUsedMd.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(1).Gauge().DataPoints().At(0).DoubleValue(), legacyMd.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(1).Gauge().DataPoints().At(0).DoubleValue(), "system.memory.utilization for the used state should be greater when computed using memAvailable")
+}

receiver/hostmetricsreceiver/internal/scraper/memoryscraper/memory_scraper_test.go

@@ -207,17 +207,21 @@ func assertMemoryUsageMetricValid(t *testing.T, metric pmetric.Metric, expectedN
 	assert.GreaterOrEqual(t, metric.Sum().DataPoints().Len(), 2)
 	internal.AssertSumMetricHasAttributeValue(t, metric, 0, "state",
 		pcommon.NewValueStr(metadata.AttributeStateUsed.String()))
+	assert.Positive(t, metric.Sum().DataPoints().At(0).IntValue())
 	internal.AssertSumMetricHasAttributeValue(t, metric, 1, "state",
 		pcommon.NewValueStr(metadata.AttributeStateFree.String()))
+	assert.Positive(t, metric.Sum().DataPoints().At(1).IntValue())
 }
 
 func assertMemoryUtilizationMetricValid(t *testing.T, metric pmetric.Metric, expectedName string) {
 	assert.Equal(t, expectedName, metric.Name())
 	assert.GreaterOrEqual(t, metric.Gauge().DataPoints().Len(), 2)
 	internal.AssertGaugeMetricHasAttributeValue(t, metric, 0, "state",
 		pcommon.NewValueStr(metadata.AttributeStateUsed.String()))
+	assert.Positive(t, metric.Gauge().DataPoints().At(0).DoubleValue())
 	internal.AssertGaugeMetricHasAttributeValue(t, metric, 1, "state",
 		pcommon.NewValueStr(metadata.AttributeStateFree.String()))
+	assert.Positive(t, metric.Gauge().DataPoints().At(1).DoubleValue())
 }
 
 func assertMemoryUsageMetricHasLinuxSpecificStateLabels(t *testing.T, metric pmetric.Metric) {