Bump the startup probe timeout for NIMs. This allows

  * Certain larger model downloads to complete whenever they are not pre-cached
  * Optimized model engine builds to complete
  * Avoid restarts while both of the above are in progress.

Signed-off-by: Shiva Krishna, Merla <smerla@nvidia.com>

Author: shivamerla

api/apps/v1alpha1/nimservice_types.go

@@ -645,7 +645,7 @@ func (n *NIMService) GetDefaultStartupProbe() *corev1.Probe {
 		TimeoutSeconds:      1,
 		PeriodSeconds:       10,
 		SuccessThreshold:    1,
-		FailureThreshold:    30,
+		FailureThreshold:    120,
 		ProbeHandler: corev1.ProbeHandler{
 			HTTPGet: &corev1.HTTPGetAction{
 				Path: "/v1/health/ready",