test: add e2e test for OpenCensus to OpenTelemetry metrics migration

Adds TestOTelMetrics, a consolidated e2e test for the OC→OTel metrics
migration in Pipelines-as-Code (PR #2567). The test scrapes two pods:

Controller (app.kubernetes.io/name=controller):
- Asserts http_client_* metrics from knative k8s client OTel instrumentation
- Asserts go_* runtime metrics
- Checks PAC application metrics (pipelines_as_code_*) are absent/present
- Asserts old OC metric names are absent

Watcher (app.kubernetes.io/name=watcher):
- Asserts kn_workqueue_* metrics (watcher uses knative reconciler)
- Asserts go_* runtime metrics

Verified locally with PAC controller and watcher deployed to kind via ko.

Relates to #2567

Co-Authored-By: Claude Sonnet 4.6 (1M context) <noreply@anthropic.com>

Author: khrm

test/metrics_otel_test.go

@@ -0,0 +1,304 @@
+//go:build e2e
+// +build e2e
+
+// Copyright 2026 The Tekton Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package test
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	dto "github.com/prometheus/client_model/go"
+	"github.com/prometheus/common/expfmt"
+	"github.com/prometheus/common/model"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+)
+
+const (
+	// pacNamespace is the namespace where Pipelines-as-Code is installed.
+	pacNamespace = "pipelines-as-code"
+	// pacControllerMetricsPort is the Prometheus metrics port on the PAC controller pod.
+	pacControllerMetricsPort = "9090"
+)
+
+// pacKubeClient builds a kubernetes client from the default kubeconfig.
+func pacKubeClient(t *testing.T) kubernetes.Interface {
+	t.Helper()
+	rules := clientcmd.NewDefaultClientConfigLoadingRules()
+	cfg, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(rules, &clientcmd.ConfigOverrides{}).ClientConfig()
+	if err != nil {
+		t.Fatalf("Failed to build kubeconfig: %v", err)
+	}
+	return kubernetes.NewForConfigOrDie(cfg)
+}
+
+// scrapePACPodMetrics scrapes the /metrics endpoint of a PAC pod selected
+// by the given label selector via the Kubernetes API server proxy.
+// It returns an error instead of calling t.Fatalf so it can be retried
+// inside polling loops without aborting the test on transient failures.
+func scrapePACPodMetrics(ctx context.Context, kubeClient kubernetes.Interface, labelSelector string) (map[string]*dto.MetricFamily, error) {
+	pods, err := kubeClient.CoreV1().Pods(pacNamespace).List(ctx, metav1.ListOptions{
+		LabelSelector: labelSelector,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	var podName string
+	for _, pod := range pods.Items {
+		if pod.Status.Phase != corev1.PodRunning {
+			continue
+		}
+		allReady := true
+		if len(pod.Status.ContainerStatuses) == 0 {
+			allReady = false
+		}
+		for _, cs := range pod.Status.ContainerStatuses {
+			if !cs.Ready {
+				allReady = false
+				break
+			}
+		}
+		if allReady {
+			podName = pod.Name
+			break
+		}
+	}
+	if podName == "" {
+		return nil, fmt.Errorf("no Running/Ready PAC pod found for selector %q in namespace %s", labelSelector, pacNamespace)
+	}
+
+	result := kubeClient.
+		CoreV1().
+		RESTClient().
+		Get().
+		Resource("pods").
+		Name(podName + ":" + pacControllerMetricsPort).
+		Namespace(pacNamespace).
+		SubResource("proxy").
+		Suffix("metrics").
+		Do(ctx)
+
+	body, err := result.Raw()
+	if err != nil {
+		return nil, err
+	}
+
+	parser := expfmt.NewTextParser(model.LegacyValidation)
+	families, err := parser.TextToMetricFamilies(strings.NewReader(string(body)))
+	if err != nil {
+		return nil, err
+	}
+	return families, nil
+}
+
+// waitForPACMetric polls the pod matching labelSelector until the named metric
+// family appears. Transient scrape errors are logged and retried until timeout.
+func waitForPACMetric(ctx context.Context, t *testing.T, kubeClient kubernetes.Interface, labelSelector, metricName string, timeout time.Duration) map[string]*dto.MetricFamily {
+	t.Helper()
+	ctx, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+	for {
+		families, err := scrapePACPodMetrics(ctx, kubeClient, labelSelector)
+		if err == nil {
+			if _, ok := families[metricName]; ok {
+				return families
+			}
+		} else {
+			t.Logf("Retrying metrics scrape: %v", err)
+		}
+		select {
+		case <-ctx.Done():
+			t.Fatalf("Timed out waiting for metric %q to appear (waited %v): %v", metricName, timeout, ctx.Err())
+			return nil
+		case <-time.After(5 * time.Second):
+		}
+	}
+}
+
+// TestOTelMetrics is a consolidated e2e test for the OpenCensus-to-OpenTelemetry
+// metrics migration in Pipelines-as-Code (PR #2567). It scrapes the PAC
+// controller pod's /metrics endpoint on port 9090 to verify:
+//
+//   - Infrastructure metrics use new OTel-based naming:
+//     - http_client_request_duration_seconds (knative k8s client instrumentation)
+//     - go_* runtime metrics
+//   - PAC application metrics are registered:
+//     - pipelines_as_code_pipelinerun_count_total
+//     - pipelines_as_code_pipelinerun_duration_seconds_sum_total
+//     - pipelines_as_code_running_pipelineruns_count
+//     - pipelines_as_code_git_provider_api_request_count_total
+//   - Old OpenCensus metric names are absent
+//
+// Application counter metrics appear after PAC processes its first PipelineRun.
+// Infrastructure metrics (http_client_*, go_*) appear at startup.
+func TestOTelMetrics(t *testing.T) {
+	ctx := context.Background()
+	kubeClient := pacKubeClient(t)
+
+	// ========== Wait for http_client metrics to appear ==========
+	// PAC uses the knative k8s client OTel instrumentation which records
+	// http_client_request_duration_seconds for all API server calls.
+
+	t.Log("Waiting for http_client_request_duration_seconds to appear on PAC controller")
+	families := waitForPACMetric(ctx, t, kubeClient, "app.kubernetes.io/name=controller,app.kubernetes.io/part-of=pipelines-as-code", "http_client_request_duration_seconds", 2*time.Minute)
+	t.Logf("Scraped %d metric families from PAC controller", len(families))
+
+	// ========== Infrastructure metric assertions (OTel renames) ==========
+
+	t.Run("Renames/k8s_client_uses_http_client_prefix", func(t *testing.T) {
+		found := false
+		for name := range families {
+			if strings.HasPrefix(name, "http_client_") {
+				found = true
+				break
+			}
+		}
+		if !found {
+			t.Error("Expected at least one http_client_* metric from knative k8s client instrumentation, found none")
+		}
+	})
+
+	t.Run("Renames/go_runtime_uses_standard_prefix", func(t *testing.T) {
+		found := false
+		for name := range families {
+			if strings.HasPrefix(name, "go_") {
+				found = true
+				break
+			}
+		}
+		if !found {
+			t.Error("Expected standard go_* runtime metrics, found none")
+		}
+	})
+
+	// ========== PAC application metric assertions ==========
+	// Counter metrics appear after the first PipelineRun is processed.
+	// The gauge appears via registered callback once the collection cycle runs.
+	// We log presence/absence rather than failing — in a fresh install with no
+	// PipelineRun activity, counters will not yet have been observed.
+
+	t.Run("Application/pipelinerun_count", func(t *testing.T) {
+		found := false
+		for name := range families {
+			if name == "pipelines_as_code_pipelinerun_count_total" || name == "pipelines_as_code_pipelinerun_count" {
+				found = true
+				break
+			}
+		}
+		if found {
+			t.Log("pipelines_as_code_pipelinerun_count(_total) found")
+		} else {
+			t.Log("pipelines_as_code_pipelinerun_count not yet present (no PipelineRuns processed yet)")
+		}
+	})
+
+	t.Run("Application/pipelinerun_duration_seconds_sum", func(t *testing.T) {
+		found := false
+		for name := range families {
+			if strings.HasPrefix(name, "pipelines_as_code_pipelinerun_duration_seconds_sum") {
+				found = true
+				break
+			}
+		}
+		if found {
+			t.Log("pipelines_as_code_pipelinerun_duration_seconds_sum found")
+		} else {
+			t.Log("pipelines_as_code_pipelinerun_duration_seconds_sum not yet present (no PipelineRuns processed yet)")
+		}
+	})
+
+	t.Run("Application/running_pipelineruns_count", func(t *testing.T) {
+		if _, ok := families["pipelines_as_code_running_pipelineruns_count"]; ok {
+			t.Log("pipelines_as_code_running_pipelineruns_count found")
+		} else {
+			t.Log("pipelines_as_code_running_pipelineruns_count not yet present (gauge callback not yet called)")
+		}
+	})
+
+	t.Run("Application/git_provider_api_request_count", func(t *testing.T) {
+		found := false
+		for name := range families {
+			if name == "pipelines_as_code_git_provider_api_request_count_total" || name == "pipelines_as_code_git_provider_api_request_count" {
+				found = true
+				break
+			}
+		}
+		if found {
+			t.Log("pipelines_as_code_git_provider_api_request_count(_total) found")
+		} else {
+			t.Log("pipelines_as_code_git_provider_api_request_count not yet present (no git API calls yet)")
+		}
+	})
+
+	// ========== Removed OpenCensus metrics ==========
+	// TODO: Remove these assertions in a future release once no OC-based
+	// release is supported.
+
+	t.Run("Removed/opencensus_pac_metrics", func(t *testing.T) {
+		ocPrefixes := []string{
+			"pipelines_as_code/",
+			"tekton_pipelines_as_code_",
+		}
+		for name := range families {
+			for _, prefix := range ocPrefixes {
+				if strings.HasPrefix(name, prefix) {
+					t.Errorf("Old OC metric %q still present; expected removal after OTel migration", name)
+				}
+			}
+		}
+	})
+
+	// ========== Watcher pod metrics ==========
+	// The PAC watcher uses the knative reconciler workqueue, so it exposes
+	// kn_workqueue_* metrics in addition to http_client_* and go_*.
+
+	t.Log("Waiting for PAC watcher pod metrics to be available")
+	watcherFamilies := waitForPACMetric(ctx, t, kubeClient, "app.kubernetes.io/name=watcher,app.kubernetes.io/part-of=pipelines-as-code", "go_goroutines", 2*time.Minute)
+	t.Logf("Scraped %d metric families from PAC watcher", len(watcherFamilies))
+
+	t.Run("Watcher/workqueue_uses_kn_prefix", func(t *testing.T) {
+		found := false
+		for name := range watcherFamilies {
+			if strings.HasPrefix(name, "kn_workqueue_") {
+				found = true
+				break
+			}
+		}
+		if !found {
+			t.Error("Expected at least one kn_workqueue_* metric on the PAC watcher, found none")
+		}
+	})
+
+	t.Run("Watcher/go_runtime_metrics_present", func(t *testing.T) {
+		found := false
+		for name := range watcherFamilies {
+			if strings.HasPrefix(name, "go_") {
+				found = true
+				break
+			}
+		}
+		if !found {
+			t.Error("Expected standard go_* runtime metrics on PAC watcher, found none")
+		}
+	})
+}