chore(e2e): update to Crunchy PostgreSQL 17 & re-enable showcase-runtime tests (#3830)

* chore(ci): update to Crunchy PostgreSQL 17

* Unskip runtime tests

* check for errors

Author: zdrapela

.ibm/pipelines/resources/postgres-db/postgres.yaml

@@ -3,8 +3,8 @@ kind: PostgresCluster
 metadata:
   name: postgress-external-db
 spec:
-  image: registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi8-16.3-1
-  postgresVersion: 16
+  image: registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi9-17.7-2547
+  postgresVersion: 17
   instances:
     - name: instance1
       dataVolumeClaimSpec:
@@ -27,7 +27,7 @@ spec:
               cpu: 200m
   backups:
     pgbackrest:
-      image: registry.developers.crunchydata.com/crunchydata/crunchy-pgbackrest:ubi8-2.51-1
+      image: registry.developers.crunchydata.com/crunchydata/crunchy-pgbackrest:ubi9-2.56.0-2547
       global:
         # Save backups for 7 days, this means 1 full backups with 6 differential ones in between
         repo1-retention-full: "1"

.ibm/pipelines/value_files/values_showcase-rbac.yaml

@@ -469,7 +469,7 @@ orchestrator:
     externalDBPort: "5432"
 
     # -- Image for the init container used by the create-db job
-    initContainerImage: "registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi8-16.3-1"
+    initContainerImage: "registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi9-17.7-2547"
 
     # -- Image for the container used by the create-db job
-    createDBJobImage: "registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi8-16.3-1"
+    createDBJobImage: "registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi9-17.7-2547"

e2e-tests/playwright/e2e/configuration-test/config-map.spec.ts

@@ -2,7 +2,7 @@ import { test, expect } from "@playwright/test";
 import { KubeClient } from "../../utils/kube-client";
 import { Common } from "../../utils/common";
 import { UIhelper } from "../../utils/ui-helper";
-test.describe.skip("Change app-config at e2e test runtime", () => {
+test.describe("Change app-config at e2e test runtime", () => {
   test.beforeAll(async () => {
     test.info().annotations.push(
       {

e2e-tests/playwright/e2e/external-database/verify-tls-config-with-external-azure-db.spec.ts

@@ -13,8 +13,7 @@ interface AzureDbConfig {
   host: string | undefined;
 }
 
-test.describe
-  .skip("Verify TLS configuration with Azure Database for PostgreSQL health check", () => {
+test.describe("Verify TLS configuration with Azure Database for PostgreSQL health check", () => {
   const namespace = process.env.NAME_SPACE_RUNTIME || "showcase-runtime";
   const job: string = process.env.JOB_NAME || "";
   let deploymentName = process.env.RELEASE_NAME + "-developer-hub";

e2e-tests/playwright/e2e/external-database/verify-tls-config-with-external-rds.spec.ts

@@ -13,8 +13,7 @@ interface RdsConfig {
   host: string | undefined;
 }
 
-test.describe
-  .skip("Verify TLS configuration with RDS PostgreSQL health check", () => {
+test.describe("Verify TLS configuration with RDS PostgreSQL health check", () => {
   const namespace = process.env.NAME_SPACE_RUNTIME || "showcase-runtime";
   const job: string = process.env.JOB_NAME || "";
   let deploymentName = process.env.RELEASE_NAME + "-developer-hub";

e2e-tests/playwright/utils/kube-client.ts

@@ -465,6 +465,87 @@ export class KubeClient {
     }
   }
 
+  /**
+   * Check if pods are in a failure state (CrashLoopBackOff, ImagePullBackOff, etc.)
+   * Returns a failure reason if found, null otherwise
+   */
+  async checkPodFailureStates(
+    namespace: string,
+    labelSelector: string,
+  ): Promise<string | null> {
+    try {
+      const response = await this.coreV1Api.listNamespacedPod(
+        namespace,
+        undefined,
+        undefined,
+        undefined,
+        undefined,
+        labelSelector,
+      );
+
+      const pods = response.body.items;
+      if (pods.length === 0) {
+        return null; // No pods yet, not a failure
+      }
+
+      for (const pod of pods) {
+        const podName = pod.metadata?.name || "unknown";
+        const phase = pod.status?.phase;
+
+        // Check for Failed phase
+        if (phase === "Failed") {
+          const reason = pod.status?.reason || "Unknown";
+          const message = pod.status?.message || "";
+          return `Pod ${podName} is in Failed phase: ${reason} - ${message}`;
+        }
+
+        // Check container statuses for failure states
+        const containerStatuses = [
+          ...(pod.status?.containerStatuses || []),
+          ...(pod.status?.initContainerStatuses || []),
+        ];
+
+        for (const containerStatus of containerStatuses) {
+          const containerName = containerStatus.name;
+          const waiting = containerStatus.state?.waiting;
+
+          if (waiting) {
+            const reason = waiting.reason || "";
+            // Check for common failure states
+            const failureStates = [
+              "CrashLoopBackOff",
+              "ImagePullBackOff",
+              "ErrImagePull",
+              "InvalidImageName",
+              "CreateContainerConfigError",
+              "CreateContainerError",
+            ];
+
+            if (failureStates.includes(reason)) {
+              const message = waiting.message || "";
+              return `Pod ${podName} container ${containerName} is in ${reason} state: ${message}`;
+            }
+          }
+
+          // Check for containers that have terminated with errors
+          const terminated = containerStatus.state?.terminated;
+          if (terminated && terminated.exitCode !== 0) {
+            const reason = terminated.reason || "Error";
+            const message = terminated.message || "";
+            console.warn(
+              `Pod ${podName} container ${containerName} terminated with exit code ${terminated.exitCode}: ${reason} - ${message}`,
+            );
+          }
+        }
+      }
+
+      return null; // No failure states detected
+    } catch (error) {
+      console.error(`Error checking pod failure states: ${error}`);
+      return null; // Don't fail the check if we can't retrieve pod info
+    }
+  }
+
   async waitForDeploymentReady(
     deploymentName: string,
     namespace: string,
@@ -492,6 +573,23 @@ export class KubeClient {
           JSON.stringify(conditions, null, 2),
         );
 
+        // Check for pod failure states when expecting replicas > 0
+        if (expectedReplicas > 0) {
+          const podFailureReason = await this.checkPodFailureStates(
+            namespace,
+            labelSelector,
+          );
+          if (podFailureReason) {
+            console.error(
+              `Pod failure detected: ${podFailureReason}. Logging events and pod logs...`,
+            );
+            await this.logDeploymentEvents(deploymentName, namespace);
+            throw new Error(
+              `Deployment ${deploymentName} failed to start: ${podFailureReason}`,
+            );
+          }
+        }
+
         // Log pod conditions using label selector
         await this.logPodConditions(namespace, labelSelector);
 
@@ -508,11 +606,17 @@ export class KubeClient {
         );
       } catch (error) {
         console.error(`Error checking deployment status: ${error}`);
+        // If we threw an error about pod failure, re-throw it
+        if (error.message?.includes("failed to start")) {
+          throw error;
+        }
       }
 
       await new Promise((resolve) => setTimeout(resolve, checkInterval));
     }
 
+    // On timeout, collect final diagnostics
+    await this.logDeploymentEvents(deploymentName, namespace);
     throw new Error(
       `Deployment ${deploymentName} did not become ready in time (timeout: ${timeout / 1000}s).`,
     );