test: add k8s test spec

distributions/nrdot-collector-host/test-spec.yaml

@@ -1,11 +1,8 @@
-fast:
-  testCaseSpecs:
-    - on_host
 slow:
   testCaseSpecs:
-    - on_host
+    - host
 nightly:
   ec2:
     enabled: true
   testCaseSpecs:
-    - on_host
+    - host

distributions/nrdot-collector-k8s/test-spec.yaml

@@ -1,11 +1,9 @@
-fast:
-  testCaseSpecs:
-    - on_host
 slow:
   testCaseSpecs:
-    - on_host
+    - k8s
+    - host
 nightly:
   ec2:
     enabled: false
   testCaseSpecs:
-    - on_host
+    - host

test/e2e/hostmetrics_nightly/hostmetrics_nightly_test.go

@@ -62,7 +62,7 @@ func TestNightly(t *testing.T) {
 					assertion := assertionFactory.NewNrMetricAssertion(testCase.Metric, testCase.Assertions)
 					// space out requests to avoid rate limiting
 					time.Sleep(time.Duration(counter) * requestSpacing)
-					assertion.ExecuteWithRetries(t, client, 15, 5*time.Second)
+					assertion.ExecuteWithRetries(t, client, 24, 5*time.Second)
 				})
 				counter += 1
 			}

test/e2e/hostmetrics_slow/hostmetrics_slow_test.go

@@ -42,15 +42,17 @@ func TestSlow(t *testing.T) {
 	client := nr.NewClient()
 
 	testEnvironment := map[string]string{
-		"hostName": testChart.NrQueryHostNamePattern,
+		"hostName":    testChart.NrQueryHostNamePattern,
+		"clusterName": kubectlOptions.ContextName,
 	}
 	for _, testCaseSpecName := range testSpec.Slow.TestCaseSpecs {
 		testCaseSpec := spec.LoadTestCaseSpec(testCaseSpecName)
 		whereClause := testCaseSpec.RenderWhereClause(testEnvironment)
+		t.Logf("test case spec where clause: %s", whereClause)
 
 		counter := 0
 		for caseName, testCase := range testCaseSpec.TestCases {
-			t.Run(fmt.Sprintf(caseName), func(t *testing.T) {
+			t.Run(fmt.Sprintf("%s/%s", testCaseSpecName, caseName), func(t *testing.T) {
 				t.Parallel()
 				assertionFactory := assert.NewNrMetricAssertionFactory(
 					whereClause,
@@ -59,7 +61,7 @@ func TestSlow(t *testing.T) {
 				assertion := assertionFactory.NewNrMetricAssertion(testCase.Metric, testCase.Assertions)
 				// space out requests to avoid rate limiting
 				time.Sleep(time.Duration(counter) * requestSpacing)
-				assertion.ExecuteWithRetries(t, client, 15, 5*time.Second)
+				assertion.ExecuteWithRetries(t, client, 24, 5*time.Second)
 			})
 			counter += 1
 		}

test/e2e/util/assert/nr_metric_assertion.go

@@ -65,7 +65,7 @@ func (m *NrMetricAssertion) ExecuteWithRetries(t testing.TB, client *newrelic.Ne
 			t.Logf("Assertion attempt %d failed with: %+v", attempt, err)
 		}
 	}
-	t.Fatalf("Assertions failed after %d attempts", retries)
+	t.Fatalf("Assertions failed after %d attempts using query: %s", retries, nrdb.NRQL(m.AsQuery()))
 }
 
 func (m *NrMetricAssertion) execute(client *newrelic.NewRelic) error {

test/e2e/util/spec/k8s.yaml

@@ -0,0 +1,14 @@
+whereClause:
+  template: "WHERE k8s.cluster.name='{{ .clusterName }}'"
+  vars:
+    - clusterName
+
+# TODO: add more, just one example test case to ensure the overall setup works
+testCases:
+  host_receiver_cpu.utilization_user:
+    metric:
+      name: k8s.pod.cpu.utilization
+    assertions:
+      - aggregationFunction: max
+        comparisonOperator: ">"
+        threshold: 0