[DagRunListener] maxtext_end_to_end chained_tests_gemma-7b_stable failed

[ml-auto-solutions-app]

• Run ID: scheduled__2025-11-13T04:00:00+00:00
• Execution Date: 2025-11-13 04:00:00+00:00
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-n2-standard-64-1.post_process.process_metrics
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-v5p-8.run_model.clean_up_workload
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-n2-standard-64-1.run_model.launch_workload.wait_for_workload_start
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-v5p-8.post_process.process_metrics

[ml-auto-solutions-app]

• Run ID: scheduled__2025-11-14T04:00:00+00:00
• Execution Date: 2025-11-14 04:00:00+00:00
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-n2-standard-64-1.post_process.process_metrics
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-v5p-8.run_model.clean_up_workload
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-n2-standard-64-1.run_model.launch_workload.wait_for_workload_start
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-v5p-8.post_process.process_metrics

[ml-auto-solutions-app]

• Run ID: scheduled__2025-11-15T04:00:00+00:00
• Execution Date: 2025-11-15 04:00:00+00:00
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-n2-standard-64-1.run_model.launch_workload.wait_for_workload_start
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-n2-standard-64-1.post_process.process_metrics
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-v5p-8.run_model.clean_up_workload
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-v5p-8.post_process.process_metrics

[ml-auto-solutions-app]

• Run ID: scheduled__2025-11-16T04:00:00+00:00
• Execution Date: 2025-11-16 04:00:00+00:00
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-n2-standard-64-1.run_model.launch_workload.wait_for_workload_start
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-n2-standard-64-1.post_process.process_metrics
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-v5p-8.run_model.clean_up_workload
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-v5p-8.post_process.process_metrics

[ml-auto-solutions-app]

• Run ID: scheduled__2025-11-17T04:00:00+00:00
• Execution Date: 2025-11-17 04:00:00+00:00
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-n2-standard-64-1.post_process.process_metrics
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-n2-standard-64-1.run_model.launch_workload.wait_for_workload_start
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-v5p-8.run_model.clean_up_workload
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-v5p-8.post_process.process_metrics

[ml-auto-solutions-app]

• Run ID: scheduled__2025-11-17T04:30:00+00:00
• Execution Date: 2025-11-17 04:30:00+00:00
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-n2-standard-64-1.post_process.process_metrics
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-n2-standard-64-1.run_model.launch_workload.wait_for_workload_start
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-v5p-8.run_model.clean_up_workload
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-v5p-8.post_process.process_metrics

[ml-auto-solutions-app]

• Run ID: scheduled__2025-11-18T04:30:00+00:00
• Execution Date: 2025-11-18 04:30:00+00:00
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-n2-standard-64-1.run_model.launch_workload.wait_for_workload_start
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-n2-standard-64-1.post_process.process_metrics
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-v5p-8.run_model.clean_up_workload
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-v5p-8.post_process.process_metrics

[ml-auto-solutions-app]

• Run ID: scheduled__2025-11-19T04:30:00+00:00
• Execution Date: 2025-11-19 04:30:00+00:00
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-n2-standard-64-1.post_process.process_metrics
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-v5p-8.run_model.clean_up_workload
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-v5p-8.post_process.process_metrics
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-n2-standard-64-1.run_model.launch_workload.wait_for_workload_start

[ml-auto-solutions-app]

• Run ID: scheduled__2025-11-20T04:30:00+00:00
• Execution Date: 2025-11-20 04:30:00+00:00
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-v5p-8.post_process.process_metrics
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-v5p-8.run_model.clean_up_workload
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-n2-standard-64-1.run_model.launch_workload.wait_for_workload_start
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-n2-standard-64-1.post_process.process_metrics

[ml-auto-solutions-app]

• Run ID: scheduled__2025-11-21T04:30:00+00:00
• Execution Date: 2025-11-21 04:30:00+00:00
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-v5p-8.run_model.clean_up_workload
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-v5p-8.post_process.process_metrics
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-n2-standard-64-1.run_model.launch_workload.wait_for_workload_start
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-n2-standard-64-1.post_process.process_metrics

[ml-auto-solutions-app]

• Run ID: scheduled__2025-11-22T04:30:00+00:00
• Execution Date: 2025-11-22 04:30:00+00:00
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-n2-standard-64-1.post_process.process_metrics
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-v5p-8.run_model.clean_up_workload
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-v5p-8.post_process.process_metrics
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-n2-standard-64-1.run_model.launch_workload.wait_for_workload_start

[ml-auto-solutions-app]

• Run ID: scheduled__2025-11-23T04:30:00+00:00
• Execution Date: 2025-11-23 04:30:00+00:00
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-n2-standard-64-1.post_process.process_metrics
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-v5p-8.run_model.clean_up_workload
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-v5p-8.post_process.process_metrics
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-n2-standard-64-1.run_model.launch_workload.wait_for_workload_start

[ml-auto-solutions-app]

• Run ID: scheduled__2025-11-24T04:30:00+00:00
• Execution Date: 2025-11-24 04:30:00+00:00
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-n2-standard-64-1.run_model.launch_workload.wait_for_workload_start
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-n2-standard-64-1.post_process.process_metrics
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-v5p-8.run_model.clean_up_workload
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-v5p-8.post_process.process_metrics

[ml-auto-solutions-app]

• Run ID: scheduled__2025-11-26T04:30:00+00:00
• Execution Date: 2025-11-26 04:30:00+00:00
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-n2-standard-64-1.run_model.launch_workload.wait_for_workload_start
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-n2-standard-64-1.post_process.process_metrics
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-v5p-8.run_model.clean_up_workload
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-v5p-8.post_process.process_metrics

[ml-auto-solutions-app]

• Run ID: scheduled__2025-11-25T04:30:00+00:00
• Execution Date: 2025-11-25 04:30:00+00:00
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-v5p-8.run_model.clean_up_workload
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-n2-standard-64-1.run_model.launch_workload.wait_for_workload_start
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-n2-standard-64-1.post_process.process_metrics
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-v5p-8.post_process.process_metrics

[ml-auto-solutions-app]

• Run ID: scheduled__2025-11-27T04:30:00+00:00
• Execution Date: 2025-11-27 04:30:00+00:00
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-n2-standard-64-1.post_process.process_metrics
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-v5p-8.run_model.clean_up_workload
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-n2-standard-64-1.run_model.launch_workload.wait_for_workload_start
• Failed Task: chained_tests_gemma-7b_stable.maxtext-stable-gemma-7b-v5p-8.post_process.process_metrics