Fix panic with Job resource diffing against an unreachable cluster (#3024)

### Proposed changes
This PR ensures that we do not make a k8s API request during the
provider's diff if there is an unreachable cluster. This currently
occurs when the Pulumi program contains a Job resource with the
`replaceUnready` annotation set to true. A panic would occur if we
attempt to make the API call since our clients are nil.

#### Testing done:
1. Created a repro test case that fails with a panic
(https://github.com/pulumi/pulumi-kubernetes/actions/runs/9228447658/job/25392833842?pr=3024)
2. Added logic to prevent the panic, and test passes subsequently
without intervention
(https://github.com/pulumi/pulumi-kubernetes/actions/runs/9228685506/job/25393667599?pr=3024)
3. Manual validation to ensure panic isn't trigerred.

### Related issues (optional)

Fixes: #3022

Author: rquitales

CHANGELOG.md

@@ -1,6 +1,7 @@
 ## Unreleased
 
 - Update to pulumi-java v0.12.0 #3025 (https://github.com/pulumi/pulumi-kubernetes/pull/3025)
+- Fixed a panic that occurs when diffing Job resources containing `replaceUnready` annotations and an unreachable cluster connection. (https://github.com/pulumi/pulumi-kubernetes/pull/3024)
 
 ## 4.12.0 (May 21, 2024)
 

provider/pkg/provider/provider.go

@@ -1743,8 +1743,12 @@ func (k *kubeProvider) Diff(ctx context.Context, req *pulumirpc.DiffRequest) (*p
 	}
 
 	if metadata.ReplaceUnready(newInputs) {
-		switch newInputs.GetKind() {
-		case "Job":
+		switch {
+		case k.clusterUnreachable:
+			// Check if the cluster is unreachable. If it is, we can't check the status of the resource otherwise
+			// a panic occurs due to the client being nil.
+			_ = k.host.Log(ctx, diag.Warning, urn, "Cluster is unreachable, skipping replaceUnready check")
+		case newInputs.GetKind() == "Job":
 			// Fetch current Job status and check point-in-time readiness. Errors are ignored.
 			if live, err := k.readLiveObject(oldLive); err == nil {
 				jobChecker := checkjob.NewJobChecker()
@@ -2677,6 +2681,7 @@ func (k *kubeProvider) gvkFromURN(urn resource.URN) (schema.GroupVersionKind, er
 
 func (k *kubeProvider) readLiveObject(obj *unstructured.Unstructured) (*unstructured.Unstructured, error) {
 	contract.Assertf(obj.GetName() != "", "expected object name to be nonempty: %v", obj)
+	contract.Assertf(k.clientSet != nil, "expected Kubernetes client-set to be non-nil")
 	rc, err := k.clientSet.ResourceClientForObject(obj)
 	if err != nil {
 		return nil, err

tests/sdk/java/testdata/job-unreachable/Pulumi.yaml

@@ -0,0 +1,32 @@
+name: job-unreachable
+runtime: yaml
+resources:
+  provider:
+    type: pulumi:providers:kubernetes
+  job:
+    type: kubernetes:batch/v1:Job
+    properties:
+      metadata:
+        name: test-job-unreachable
+        annotations:
+          pulumi.com/replaceUnready: "true"
+      spec:
+        template:
+          metadata:
+            name: test-job-unreachable
+          spec:
+            containers:
+              - name: test-job-unreachable-container
+                image: busybox
+                # This command will cause the container to exit with a non-zero status code, and fail the job.
+                command:
+                  - sh
+                  - -c
+                  - exit 1
+            restartPolicy: Never
+    options:
+      provider: ${provider}
+      customTimeouts:
+        create: 15s
+        update: 15s
+        delete: 15s

tests/sdk/java/testdata/job-unreachable/step2/Pulumi.yaml

@@ -0,0 +1,34 @@
+name: job-unreachable
+runtime: yaml
+resources:
+  provider:
+    type: pulumi:providers:kubernetes
+    properties:
+      kubeconfig: "fake-kubeconfig-data"
+  job:
+    type: kubernetes:batch/v1:Job
+    properties:
+      metadata:
+        name: test-job-unreachable
+        annotations:
+          pulumi.com/replaceUnready: "true"
+      spec:
+        template:
+          metadata:
+            name: test-job-unreachable
+          spec:
+            containers:
+              - name: test-job-unreachable-container
+                image: busybox
+                # This command will cause the container to exit with a non-zero status code, and fail the job.
+                command:
+                  - sh
+                  - -c
+                  - exit 1
+            restartPolicy: Never
+    options:
+      provider: ${provider}
+      customTimeouts:
+        create: 15s
+        update: 15s
+        delete: 15s

tests/sdk/java/yamlv2_test.go

@@ -4,6 +4,7 @@ import (
 	"testing"
 
 	"github.com/pulumi/providertest/pulumitest"
+	"github.com/stretchr/testify/assert"
 )
 
 // TestYamlV2 deploys a complex stack using yaml/v2 package.
@@ -21,3 +22,23 @@ func TestYamlV2(t *testing.T) {
 	test.Preview()
 	test.Up()
 }
+
+// TestJobUnreachable ensures that a panic does not occur when diffing Job resources against an unreachable API server.
+// https://github.com/pulumi/pulumi-kubernetes/issues/3022
+func TestJobUnreachable(t *testing.T) {
+	test := pulumitest.NewPulumiTest(t, "testdata/job-unreachable")
+	t.Logf("into %s", test.Source())
+	t.Cleanup(func() {
+		test.Destroy()
+	})
+	test.Preview()
+
+	// Create the job, but expect it to fail as the job is meant to fail.
+	_, err := test.CurrentStack().Up(test.Context())
+	assert.ErrorContains(t, err, `but the Kubernetes API server reported that it failed to fully initialize or become live`)
+
+	// Re-run the Pulumi program with a malformed kubeconfig to simulate an unreachable API server.
+	// This should not panic annd preview should succeed.
+	test.UpdateSource("testdata/job-unreachable/step2")
+	test.Preview()
+}