fix

Author: zhengkezhou1

.github/workflows/gke-connectivity-smoke.yml

@@ -227,7 +227,7 @@ jobs:
             completions: 4
             backoffLimit: 0
             activeDeadlineSeconds: 7200
-            ttlSecondsAfterFinished: 600
+            ttlSecondsAfterFinished: 3600
             template:
               metadata:
                 annotations:
@@ -239,7 +239,7 @@ jobs:
                 nodeSelector:
                   cloud.google.com/gke-tpu-accelerator: tpu-v6e-slice
                   cloud.google.com/gke-tpu-topology: 4x4
-                  # cloud.google.com/gke-nodepool: ${NODE_POOL}
+                  cloud.google.com/gke-nodepool: ${NODE_POOL}
                 tolerations:
                 - key: google.com/tpu
                   operator: Equal
@@ -326,7 +326,7 @@ jobs:
                     readOnly: true
                     volumeAttributes:
                       bucketName: ${BUCKET_NAME}
-                      mountOptions: implicit-dirs,metadata-cache:ttl-secs:-1,metadata-cache:stat-cache-max-size-mb:-1,metadata-cache:type-cache-max-size-mb:-1,file-cache:max-size-mb:409600,file-cache:cache-file-for-range-read:true,file-system:kernel-list-cache-ttl-secs:-1,file-cache:enable-parallel-downloads:true,file-cache:parallel-downloads-per-file:128,read_ahead_kb=1024
+                      mountOptions: implicit-dirs,metadata-cache:ttl-secs:-1,metadata-cache:stat-cache-max-size-mb:-1,metadata-cache:type-cache-max-size-mb:-1,file-cache:max-size-mb:0,file-system:kernel-list-cache-ttl-secs:-1,read_ahead_kb=1024
           EOF
 
           kubectl apply -f /tmp/tpu-smoke.yaml
@@ -341,9 +341,28 @@ jobs:
             --prefix=true \
             --follow=true \
             --max-log-requests=8
-          kubectl wait --for=condition=complete "job/${WORKLOAD_NAME}" \
-            --namespace "${NAMESPACE}" \
-            --timeout="${JOB_COMPLETE_TIMEOUT}"
+
+          # Wait for Job to reach a terminal state (Complete or Failed). Plain
+          # `kubectl wait --for=condition=complete` only matches success and
+          # would block until --timeout even if the Job has already Failed.
+          deadline=$(($(date +%s) + 1800))
+          while [[ $(date +%s) -lt $deadline ]]; do
+            conds=$(kubectl get job "${WORKLOAD_NAME}" --namespace "${NAMESPACE}" \
+              -o jsonpath='{.status.conditions[?(@.type=="Complete")].status}|{.status.conditions[?(@.type=="Failed")].status}')
+            case "${conds}" in
+              True\|*)
+                echo "Job completed successfully"
+                exit 0
+                ;;
+              \|True)
+                echo "Job failed (condition=Failed). See diagnose step for details."
+                exit 1
+                ;;
+            esac
+            sleep 10
+          done
+          echo "Timed out waiting for Job to reach a terminal state"
+          exit 1
 
       - name: Diagnose TPU workload failure
         if: failure()
@@ -353,6 +372,11 @@ jobs:
 
           kubectl get pods --namespace "${NAMESPACE}" --selector="job-name=${WORKLOAD_NAME}" -o wide || true
           kubectl describe pods --namespace "${NAMESPACE}" --selector="job-name=${WORKLOAD_NAME}" || true
+          kubectl describe job "${WORKLOAD_NAME}" --namespace "${NAMESPACE}" || true
+          kubectl get events --namespace "${NAMESPACE}" --sort-by='.lastTimestamp' \
+            --field-selector="involvedObject.name=${WORKLOAD_NAME}" || true
+          kubectl get events --namespace "${NAMESPACE}" --sort-by='.lastTimestamp' \
+            | grep -E "${WORKLOAD_NAME}|Evicted|DiskPressure|MemoryPressure|OOMKill|FailedScheduling" || true
           kubectl logs --namespace "${NAMESPACE}" --selector="job-name=${WORKLOAD_NAME}" --all-containers=true --prefix=true || true
 
       - name: Cleanup TPU workload smoke test