|
| 1 | +// Package e2e contains end-to-end tests for the CoreDNS probe. |
| 2 | +package e2e |
| 3 | + |
| 4 | +import ( |
| 5 | + "context" |
| 6 | + "fmt" |
| 7 | + "io" |
| 8 | + "maps" |
| 9 | + "net/http" |
| 10 | + "os" |
| 11 | + "os/exec" |
| 12 | + "path/filepath" |
| 13 | + "slices" |
| 14 | + "strings" |
| 15 | + "testing" |
| 16 | + |
| 17 | + . "github.com/onsi/ginkgo/v2" |
| 18 | + . "github.com/onsi/gomega" |
| 19 | + "github.com/onsi/gomega/gbytes" |
| 20 | + "github.com/onsi/gomega/gexec" |
| 21 | + "github.com/prometheus/common/expfmt" |
| 22 | + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" |
| 23 | + "k8s.io/client-go/kubernetes" |
| 24 | + "k8s.io/client-go/tools/clientcmd" |
| 25 | +) |
| 26 | + |
| 27 | +func TestE2E(t *testing.T) { |
| 28 | + RegisterFailHandler(Fail) |
| 29 | + RunSpecs(t, "CoreDNS Probe E2E Suite") |
| 30 | +} |
| 31 | + |
| 32 | +const ( |
| 33 | + clusterName = "corednsprobe-test" |
| 34 | + namespace = "kube-system" |
| 35 | + deploymentName = "coredns-probe" |
| 36 | + metricsPort = 9091 |
| 37 | + probeImage = "paulgmiller/corednsprobe:e2etest" |
| 38 | +) |
| 39 | + |
| 40 | +var ( |
| 41 | + clientset *kubernetes.Clientset |
| 42 | + testDir string |
| 43 | + corednsIPs map[string]struct{} |
| 44 | +) |
| 45 | + |
| 46 | +var _ = BeforeSuite(func() { |
| 47 | + // Create a temporary directory for test artifacts. |
| 48 | + testDir, err := os.MkdirTemp("", "corednsprobe-e2e-") |
| 49 | + Expect(err).NotTo(HaveOccurred()) |
| 50 | + |
| 51 | + By("Creating a Kind cluster") |
| 52 | + kubeConfigPath := filepath.Join(testDir, "kubeconfig") |
| 53 | + os.Setenv("KUBECONFIG", kubeConfigPath) |
| 54 | + kindCmd := exec.Command("kind", "create", "cluster", "--name", clusterName, "--kubeconfig", kubeConfigPath) |
| 55 | + output, err := kindCmd.CombinedOutput() |
| 56 | + Expect(err).NotTo(HaveOccurred(), "Failed to create Kind cluster: %s", string(output)) |
| 57 | + GinkgoWriter.Println(string(output)) |
| 58 | + |
| 59 | + // Initialize Kubernetes client. |
| 60 | + config, err := clientcmd.BuildConfigFromFlags("", kubeConfigPath) |
| 61 | + Expect(err).NotTo(HaveOccurred()) |
| 62 | + clientset, err = kubernetes.NewForConfig(config) |
| 63 | + Expect(err).NotTo(HaveOccurred()) |
| 64 | + |
| 65 | + By("Building Docker image for CoreDNS probe") |
| 66 | + gitRoot, err := getGitRoot() |
| 67 | + Expect(err).NotTo(HaveOccurred(), "Failed to get Git root directory") |
| 68 | + buildCmd := exec.Command("docker", "build", "-t", probeImage, gitRoot) |
| 69 | + buildOutput, err := buildCmd.CombinedOutput() |
| 70 | + Expect(err).NotTo(HaveOccurred(), "Failed to build Docker image: %s", string(buildOutput)) |
| 71 | + GinkgoWriter.Println(string(buildOutput)) |
| 72 | + |
| 73 | + By("Loading Docker image into Kind") |
| 74 | + loadCmd := exec.Command("kind", "load", "docker-image", probeImage, "--name", clusterName) |
| 75 | + loadOutput, err := loadCmd.CombinedOutput() |
| 76 | + Expect(err).NotTo(HaveOccurred(), "Failed to load image into Kind: %s", string(loadOutput)) |
| 77 | + GinkgoWriter.Println(string(loadOutput)) |
| 78 | + |
| 79 | + By("Waiting for CoreDNS pods to be running") |
| 80 | + corednsIPs = make(map[string]struct{}) |
| 81 | + Eventually(func() bool { |
| 82 | + podList, err := clientset.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{ |
| 83 | + LabelSelector: "k8s-app=kube-dns", |
| 84 | + }) |
| 85 | + if err != nil { |
| 86 | + return false |
| 87 | + } |
| 88 | + for _, pod := range podList.Items { |
| 89 | + if pod.Status.Phase != "Running" || pod.Status.PodIP == "" { |
| 90 | + return false |
| 91 | + } |
| 92 | + corednsIPs[pod.Status.PodIP] = struct{}{} |
| 93 | + } |
| 94 | + return len(podList.Items) > 0 |
| 95 | + }, "180s", "2s").Should(BeTrue(), "CoreDNS pods are not running") |
| 96 | + GinkgoWriter.Println("CoreDNS pod IPs:", slices.Collect(maps.Keys(corednsIPs))) |
| 97 | + |
| 98 | + By("Deploying CoreDNS probe") |
| 99 | + deployCmdStr := fmt.Sprintf("kustomize edit set image %s && kustomize build . | kubectl apply -f -", probeImage) |
| 100 | + deployCmd := exec.Command("bash", "-c", deployCmdStr) |
| 101 | + deployCmd.Env = os.Environ() |
| 102 | + deployCmd.Dir = filepath.Join(gitRoot, "config", "overlays", "e2e") |
| 103 | + deployOutput, err := deployCmd.CombinedOutput() |
| 104 | + Expect(err).NotTo(HaveOccurred(), "Failed to deploy CoreDNS probe: %s", string(deployOutput)) |
| 105 | + GinkgoWriter.Println(string(deployOutput)) |
| 106 | + |
| 107 | + By("Waiting for CoreDNS probe deployment to become ready") |
| 108 | + Eventually(func() bool { |
| 109 | + deployment, err := clientset.AppsV1().Deployments(namespace).Get(context.TODO(), deploymentName, metav1.GetOptions{}) |
| 110 | + if err != nil { |
| 111 | + return false |
| 112 | + } |
| 113 | + return deployment.Status.ReadyReplicas == *deployment.Spec.Replicas |
| 114 | + }, "90s", "2s").Should(BeTrue()) |
| 115 | + |
| 116 | + By("Listing all pods in all namespaces") |
| 117 | + podsCmd := exec.Command("kubectl", "get", "po", "-A") |
| 118 | + podsCmd.Env = os.Environ() |
| 119 | + podsOutput, err := podsCmd.CombinedOutput() |
| 120 | + Expect(err).NotTo(HaveOccurred(), "Failed to list pods: %s", string(podsOutput)) |
| 121 | + GinkgoWriter.Println(string(podsOutput)) |
| 122 | +}) |
| 123 | + |
| 124 | +var _ = AfterSuite(func() { |
| 125 | + By("Deleting the Kind cluster") |
| 126 | + kindCmd := exec.Command("kind", "delete", "cluster", "--name", clusterName) |
| 127 | + kindCmd.CombinedOutput() |
| 128 | + |
| 129 | + os.RemoveAll(testDir) |
| 130 | +}) |
| 131 | + |
| 132 | +var _ = Describe("CoreDNS Probe deployment", func() { |
| 133 | + It("should have the CoreDNS probe pod running", func() { |
| 134 | + deployment, err := clientset.AppsV1().Deployments(namespace).Get(context.TODO(), deploymentName, metav1.GetOptions{}) |
| 135 | + Expect(err).NotTo(HaveOccurred()) |
| 136 | + Expect(deployment.Status.AvailableReplicas).To(Equal(*deployment.Spec.Replicas)) |
| 137 | + |
| 138 | + podList, err := clientset.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{ |
| 139 | + LabelSelector: "app=" + deploymentName, |
| 140 | + }) |
| 141 | + Expect(err).NotTo(HaveOccurred()) |
| 142 | + Expect(podList.Items).NotTo(BeEmpty()) |
| 143 | + }) |
| 144 | + |
| 145 | + It("should expose metrics endpoint", func() { |
| 146 | + podList, err := clientset.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{ |
| 147 | + LabelSelector: "app=" + deploymentName, |
| 148 | + }) |
| 149 | + Expect(err).NotTo(HaveOccurred()) |
| 150 | + Expect(podList.Items).NotTo(BeEmpty()) |
| 151 | + |
| 152 | + pod := podList.Items[0] |
| 153 | + |
| 154 | + By("Port-forwarding to the CoreDNS probe pod") |
| 155 | + portForwardCmd := exec.Command("kubectl", "port-forward", |
| 156 | + fmt.Sprintf("pod/%s", pod.Name), |
| 157 | + fmt.Sprintf("%d:%d", metricsPort, metricsPort), |
| 158 | + "-n", namespace) |
| 159 | + portForwardCmd.Env = os.Environ() |
| 160 | + session, err := gexec.Start(portForwardCmd, GinkgoWriter, GinkgoWriter) |
| 161 | + Expect(err).NotTo(HaveOccurred()) |
| 162 | + defer session.Kill() |
| 163 | + |
| 164 | + By("Waiting for port forwarding to be established") |
| 165 | + Eventually(session, "5s", "1s").Should(gbytes.Say("Forwarding from"), "Failed to establish port-forwarding") |
| 166 | + |
| 167 | + By("Checking if metrics endpoint is accessible") |
| 168 | + res, err := http.Get(fmt.Sprintf("http://localhost:%d/metrics", metricsPort)) |
| 169 | + Expect(err).NotTo(HaveOccurred(), "Failed to access metrics endpoint") |
| 170 | + defer res.Body.Close() |
| 171 | + Expect(res.StatusCode).To(Equal(http.StatusOK), "Metrics endpoint did not return 200 OK") |
| 172 | + |
| 173 | + By("Verifying metrics format") |
| 174 | + body, err := io.ReadAll(res.Body) |
| 175 | + Expect(err).NotTo(HaveOccurred(), "Failed to read response body") |
| 176 | + Expect(body).NotTo(BeEmpty(), "Metrics response body is empty") |
| 177 | + var parser expfmt.TextParser |
| 178 | + metrics, err := parser.TextToMetricFamilies(strings.NewReader(string(body))) |
| 179 | + Expect(err).NotTo(HaveOccurred(), "Failed to parse metrics") |
| 180 | + metric := metrics["coredns_probe_rtt_milliseconds"] |
| 181 | + Expect(metric).NotTo(BeNil(), "Expected coredns_probe_rtt_milliseconds metric not found") |
| 182 | + |
| 183 | + By("Verifying metrics endpoint labels match CoreDNS IPs") |
| 184 | + Expect(corednsIPs).NotTo(BeEmpty(), "No CoreDNS pod IPs were discovered") |
| 185 | + metricEndpoints := make(map[string]struct{}) |
| 186 | + for _, m := range metric.Metric { |
| 187 | + for _, label := range m.Label { |
| 188 | + if label.GetName() == "endpoint" { |
| 189 | + ip := label.GetValue() |
| 190 | + _, exists := corednsIPs[ip] |
| 191 | + Expect(exists).To(BeTrue(), fmt.Sprintf("Unexpected endpoint in metrics: %s", ip)) |
| 192 | + metricEndpoints[ip] = struct{}{} |
| 193 | + GinkgoWriter.Println("Found metrics for CoreDNS IP:", ip) |
| 194 | + break |
| 195 | + } |
| 196 | + } |
| 197 | + } |
| 198 | + Expect(maps.Equal(metricEndpoints, corednsIPs)).To(BeTrue(), "Metrics endpoints don't match CoreDNS IPs") |
| 199 | + }) |
| 200 | +}) |
| 201 | + |
| 202 | +// getGitRoot retrieves the root directory of the Git repository. |
| 203 | +func getGitRoot() (string, error) { |
| 204 | + cmd := exec.Command("git", "rev-parse", "--show-toplevel") |
| 205 | + output, err := cmd.CombinedOutput() |
| 206 | + if err != nil { |
| 207 | + return "", fmt.Errorf("failed to get Git root directory: %w", err) |
| 208 | + } |
| 209 | + |
| 210 | + return strings.TrimSpace(string(output)), nil |
| 211 | +} |
0 commit comments