Testing consul-1.21-dev + consul-k8s-1.7.x with ocp-4.16

- Testing consul-1.21-dev + consul-k8s-1.7.x  with ocp-4.16 + k8s-1.30.0 + kubectl 1.30.0
- Added OCP logging for error debugging and cluster stuck issues and finalizer issues

Author: anandmukul93

.github/workflows/pr-cloud-accepance.yml

@@ -0,0 +1,25 @@
+# Dispatch to the consul-k8s-workflows with a nightly cron
+name: pr-openshift-acceptance
+on:
+  pull_request:
+    branches:
+      - release/1.7.0-rc1
+
+# these should be the only settings that you will ever need to change
+env:
+  BRANCH: ${{ github.event.pull_request.head.ref }}
+  CONTEXT: "pr"
+
+jobs:
+  cloud-acceptance:
+    name: cloud-acceptance
+    runs-on: ubuntu-latest
+    steps:
+      - uses: benc-uk/workflow-dispatch@25b02cc069be46d637e8fe2f1e8484008e9e9609 # v1.2.3
+        name: cloud
+        with:
+          workflow: cloud.yml
+          repo: hashicorp/consul-k8s-workflows
+          ref: mukul/testing-ocp-compatibility
+          token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
+          inputs: '{ "context":"${{ env.CONTEXT }}", "repository":"${{ github.repository }}", "branch":"${{ env.BRANCH }}", "sha":"${{ github.sha }}", "token":"${{ secrets.ELEVATED_GITHUB_TOKEN }}", "test-integrations": "eks, aks, gke" }'

.github/workflows/pr.yml

@@ -2,6 +2,8 @@
 name: pr
 on:
   pull_request:
+    branches:
+      - release/1.7.0-rc1
 
 # these should be the only settings that you will ever need to change
 env:
@@ -24,7 +26,7 @@ jobs:
       with:
         workflow: test.yml
         repo: hashicorp/consul-k8s-workflows
-        ref: main
+        ref: mukul/testing-ocp-compatibility
         token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
         inputs: '{ "context":"${{ env.CONTEXT }}", "actor":"${{ github.actor }}", "repository":"${{ github.repository }}", "branch":"${{ env.BRANCH }}", "sha":"${{ env.SHA }}", "token":"${{ secrets.ELEVATED_GITHUB_TOKEN }}" }'
 

.github/workflows/weekly-acceptance-1-7-x.yml

@@ -6,7 +6,7 @@ on:
   schedule:
     # * is a special character in YAML so you have to quote this string
     # Run weekly on Sunday at 3AM UTC/11PM EST/8PM PST
-    - cron:  '0 3 * * 7'
+    - cron:  '0 3 * * 0'
 
 # these should be the only settings that you will ever need to change
 env:

acceptance/ci-inputs/kind-inputs.yaml

@@ -2,5 +2,6 @@
 # SPDX-License-Identifier: MPL-2.0
 
 kindVersion: v0.23.0
-kindNodeImage: kindest/node:v1.30.2@sha256:ecfe5841b9bee4fe9690f49c118c33629fa345e3350a0c67a5a34482a99d6bba
-kubectlVersion: v1.30.2
+# digest for kindest/node:v1.30.0
+kindNodeImage: kindest/node:v1.30.0
+kubectlVersion: v1.30.0

acceptance/tests/openshift/openshift_test_runner.go

@@ -5,15 +5,148 @@ import (
 	"github.com/hashicorp/consul-k8s/acceptance/framework/helpers"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"os"
 	"os/exec"
 	"strconv"
+	"strings"
+	"syscall"
 	"testing"
 )
 
+func interruptProcess(t *testing.T) {
+	p, err := os.FindProcess(os.Getpid())
+	if err != nil {
+		t.Logf("Failed to find process: %v", err)
+		return
+	}
+	err = p.Signal(syscall.SIGINT)
+	if err != nil {
+		t.Logf("Failed to send interrupt signal: %v", err)
+	}
+}
+func removeNamespaceFinalizer(t *testing.T, namespace string) {
+	cmd := exec.Command("kubectl", "patch", "namespace", namespace,
+		"--type=json", "-p", `[{"op": "remove", "path": "/spec/finalizers"}]`)
+
+	if output, err := cmd.CombinedOutput(); err != nil {
+		t.Logf("Error removing namespace finalizer: %v\nOutput: %s\n", err, output)
+	}
+	verifyNamespaceDeletion(t, namespace)
+}
+
+func verifyNamespaceDeletion(t *testing.T, namespace string) {
+	checkCmd := exec.Command("kubectl", "get", "ns", namespace)
+	if checkErr := checkCmd.Run(); checkErr != nil {
+		t.Logf("Namespace %s deleted successfully\n", namespace)
+		return
+	}
+
+	t.Logf("Namespace still exists. Additional cleanup might be required")
+	interruptProcess(t)
+}
+
+func gatewayControllerDiagnostic(t *testing.T, namespace string) {
+	// Add diagnostic logging for pods for controller identification
+	lsNamespaceCmd := exec.Command("oc", "get", "namespaces", "-o", "wide")
+	lsNamespaceOutput, _ := lsNamespaceCmd.CombinedOutput()
+	t.Logf("Namespaces in cluster:\n%s", string(lsNamespaceOutput))
+
+	podInfoCmd := exec.Command("kubectl", "get", "pods", "-n", namespace)
+	podInfoOutput, _ := podInfoCmd.CombinedOutput()
+	t.Logf("Pod status in namespace %s before cleanup:\n%s", namespace, string(podInfoOutput))
+
+	podInfoCmd = exec.Command("kubectl", "get", "pods", "-n", "kube-system")
+	podInfoOutput, _ = podInfoCmd.CombinedOutput()
+	t.Logf("Pod status in namespace %s before cleanup:\n%s", "kube-system", string(podInfoOutput))
+}
+
+func checkAndDeleteNamespace(t *testing.T) {
+	// There are some commands in this function
+	//which are not being replaced with this variables and need manual replacement of namespace
+	namespace := "consul"
+	// Check if namespace exists and its status
+	nsCheckCmd := exec.Command("kubectl", "get", "namespace", namespace, "-o", "json")
+	nsOutput, _ := nsCheckCmd.CombinedOutput()
+	t.Logf("Consul namespace status before cleanup:\n%s", string(nsOutput))
+
+	// Add diagnostic logging before attempting cleanup
+	logCmd := exec.Command("kubectl", "get", "all", "-n", namespace)
+	logOutput, _ := logCmd.CombinedOutput()
+	t.Logf("Resources in consul namespace before cleanup:\n%s", string(logOutput))
+
+	// find gateway controller information and logs
+	gatewayControllerDiagnostic(t, namespace)
+	// Force cleanup of any stuck resources in the namespace (if it still exists)
+	t.Log("Checking for any stuck resources...")
+	forceCleanupCmd := exec.Command("bash", "-c", `
+		# Try to find finalizers on the namespace
+		FINALIZERS=$(kubectl get namespace consul -o json 2>/dev/null | jq '.spec.finalizers' 2>/dev/null)
+		if [ ! -z "$FINALIZERS" ] && [ "$FINALIZERS" != "null" ]; then
+			echo "Found finalizers on namespace consul"
+			echo $FINALIZERS
+			# Remove finalizers from namespace to force deletion
+			# kubectl get namespace consul -o json | jq '.spec.finalizers = []' | kubectl replace --raw "/api/v1/namespaces/consul/finalize" -f -
+		fi
+		if kubectl get namespace consul > /dev/null 2>&1; then
+			# Check for gateway resources
+			GATEWAYS=$(kubectl get gateways.gateway.networking.k8s.io -n consul -o json 2>/dev/null || echo "")
+			echo $GATEWAYS
+		fi
+	`)
+	forceOutput, _ := forceCleanupCmd.CombinedOutput()
+	t.Logf("Force cleanup result:\n%s", string(forceOutput))
+
+	// Get remaining Gateways
+	getCmd := exec.Command("kubectl", "get", "gateways.gateway.networking.k8s.io", "-n", namespace,
+		"-o=jsonpath={.items[*].metadata.name}")
+	output, err := getCmd.CombinedOutput()
+	t.Logf("Gateway resource check result:\n%s", string(output))
+
+	if err != nil {
+		t.Logf("Error getting gateways: %v\n", err)
+		return
+	}
+	cleanedOutput := strings.TrimSpace(string(output))
+	if cleanedOutput == "" {
+		t.Logf("No gateways found, removing namespace finalizer")
+		removeNamespaceFinalizer(t, namespace)
+		return
+	}
+	if len(cleanedOutput) > 0 {
+		// Remove finalizers from each gateway
+		patchCmd := exec.Command("kubectl", "patch", "gateways.gateway.networking.k8s.io", string(output), "-n", namespace,
+			"--type=json", "-p", `[{"op": "remove", "path": "/metadata/finalizers"}]`)
+		patchOutput, patchErr := patchCmd.CombinedOutput()
+		if patchErr != nil {
+			t.Logf("Error patching gateway: %v\nOutput: %s\n", patchErr, patchOutput)
+			return
+		}
+		t.Logf("Finalizers removed successfully")
+		removeNamespaceFinalizer(t, namespace)
+	}
+	t.Log("Attempting to delete consul namespace if it exists...")
+	cleanupCmd := exec.Command("kubectl", "delete", "namespace", "consul", "--ignore-not-found=true")
+	cleanupOutput, cleanupErr := cleanupCmd.CombinedOutput()
+	// We don't check error here since it's just precautionary cleanup
+	t.Logf("Namespace deletion attempt result: %v\nOutput: %s", cleanupErr, string(cleanupOutput))
+
+	// Wait for namespace to be fully deleted before proceeding
+	t.Log("Waiting for consul namespace to be fully deleted...")
+	waitCmd := exec.Command("kubectl", "wait", "--for=delete", "namespace/consul", "--timeout=30s")
+	waitOutput, waitErr := waitCmd.CombinedOutput() // Ignore errors, as this will error if the namespace doesn't exist at all
+	t.Logf("Wait result: %v\nOutput: %s", waitErr, string(waitOutput))
+
+	// Verify namespace deletion
+	verifyNamespaceDeletion(t, namespace)
+}
+
 func newOpenshiftCluster(t *testing.T, cfg *config.TestConfig, secure, namespaceMirroring bool) {
 	cmd := exec.Command("helm", "repo", "add", "hashicorp", "https://helm.releases.hashicorp.com")
 	output, err := cmd.CombinedOutput()
-	require.NoErrorf(t, err, "failed to add hashicorp helm repo: %s", string(output))
+	require.NoErrorf(t, err, "failed to add hashicorp helm repo : %s", string(output))
+
+	// Check for any stuck resources in the namespace and force cleanup if necessary
+	checkAndDeleteNamespace(t)
 
 	// FUTURE for some reason NewHelmCluster creates a consul server pod that runs as root which
 	//   isn't allowed in OpenShift. In order to test OpenShift properly, we have to call helm and k8s
@@ -26,7 +159,7 @@ func newOpenshiftCluster(t *testing.T, cfg *config.TestConfig, secure, namespace
 		assert.NoErrorf(t, err, "failed to delete namespace: %s", string(output))
 	})
 
-	require.NoErrorf(t, err, "failed to add hashicorp helm repo: %s", string(output))
+	require.NoErrorf(t, err, "failed to create namespace: %s", string(output))
 
 	cmd = exec.Command("kubectl", "create", "secret", "generic",
 		"consul-ent-license",
@@ -62,6 +195,7 @@ func newOpenshiftCluster(t *testing.T, cfg *config.TestConfig, secure, namespace
 	)
 
 	output, err = cmd.CombinedOutput()
+	t.Logf("Output of the helm install command: %s", string(output))
 	helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() {
 		cmd := exec.Command("helm", "uninstall", "consul", "--namespace", "consul")
 		output, err := cmd.CombinedOutput()

charts/consul/test/terraform/aks/main.tf

@@ -11,6 +11,9 @@ terraform {
 
 provider "azurerm" {
   features {}
+  api_version_override = {
+    "managedClusters" = "2025-01-01"  # Use one of the supported versions from the error message
+  }
 }
 
 provider "local" {}