Refine PodFailureSpec (#42)

Author: yqwang-ms

README.md

@@ -52,10 +52,10 @@ A Framework represents an application with a set of Tasks:
 5. Support to specify how to [classify and summarize Pod failures](doc/user-manual.md#PodFailureClassification)
 6. Support to expose [Framework and Pod history snapshots](doc/user-manual.md#FrameworkPodHistory) to external systems
 7. Easy to leverage [FrameworkBarrier](doc/user-manual.md#FrameworkBarrier) to achieve light-weight Gang Execution and Service Discovery
-8. Easy to leverage [HivedScheduler](doc/user-manual.md#HivedScheduler) to achieve GPU Multi-Tenant, Topology-Aware, Priority and Gang Scheduling
+8. Easy to leverage [HivedScheduler](doc/user-manual.md#HivedScheduler) to achieve GPU Topology-Aware, Multi-Tenant, Priority and Gang Scheduling
 9. Compatible with other Kubernetes features, such as Kubernetes [Service](https://kubernetes.io/docs/concepts/services-networking/service), [Gpu Scheduling](https://kubernetes.io/docs/tasks/manage-gpus/scheduling-gpus), [Volume](https://kubernetes.io/docs/concepts/storage/volumes/), [Logging](https://kubernetes.io/docs/concepts/cluster-administration/logging)
 10. Idiomatic with Kubernetes official controllers, such as [Pod Spec](https://kubernetes.io/docs/concepts/workloads/pods/pod-overview/#pod-templates)
-11. Aligned with Kubernetes [Controller Design Guidelines](https://github.com/kubernetes/community/blob/master/contributors/devel/controllers.md) and [API Conventions](https://github.com/kubernetes/community/blob/master/contributors/devel/api-conventions.md)
+11. Aligned with Kubernetes [Controller Design Guidelines](https://github.com/kubernetes/community/blob/f0dd87ad477e1e91c53866902adf7832c32ce543/contributors/devel/sig-api-machinery/controllers.md) and [API Conventions](https://github.com/kubernetes/community/blob/a2cdce51a0bbbc214f0e8813e0a877176ad3b6c9/contributors/devel/sig-architecture/api-conventions.md)
 
 ## Prerequisite
 1. A Kubernetes cluster, v1.14.2 or above, on-cloud or on-premise.

doc/user-manual.md

@@ -202,6 +202,8 @@ Watch the change events of all Frameworks (in the specified FrameworkNamespace).
 ## <a name="PodFailureClassification">Pod Failure Classification</a>
 You can specify how to classify and summarize Pod failures by [PodFailureSpec](../pkg/apis/frameworkcontroller/v1/config.go).
 
+You can also directly leverage the [Default PodFailureSpec](../example/config/default/frameworkcontroller.yaml).
+
 ## <a name="PredefinedCompletionCode">Predefined CompletionCode</a>
 You can leverage the [Predefined CompletionCode](../pkg/apis/frameworkcontroller/v1/completion.go) to instruct your [RetryPolicy](#RetryPolicy) and identify a certain predefined CompletionCode, regardless of different [PodFailureSpec](../pkg/apis/frameworkcontroller/v1/config.go) may be configured in different clusters.
 

example/config/default/frameworkcontroller.yaml

@@ -81,17 +81,15 @@ podFailureSpec:
       nameRegex: '(?ms).*'
       messageRegex: '(?ms).*'
 - code: -1201
-  phrase: ContainerDockerRunFlagInvalid
-  type:
-    attributes: [Permanent]
+  phrase: ContainerDockerCreationError
   podPatterns:
   - containers:
     - reasonRegex: '(?i)^ContainerCannotRun$'
       codeRange: {min: 125, max: 125}
       nameRegex: '(?ms).*'
       messageRegex: '(?ms).*'
 - code: -1202
-  phrase: ContainerDockerRunPermissionDenied
+  phrase: ContainerDockerStartCmdPermissionDenied
   type:
     attributes: [Permanent]
   podPatterns:
@@ -101,7 +99,7 @@ podFailureSpec:
       nameRegex: '(?ms).*'
       messageRegex: '(?ms).*'
 - code: -1203
-  phrase: ContainerDockerRunCmdNotFound
+  phrase: ContainerDockerStartCmdNotFound
   type:
     attributes: [Permanent]
   podPatterns:
@@ -116,7 +114,7 @@ podFailureSpec:
       nameRegex: '(?ms).*'
       messageRegex: '(?msi).*(not found|cannot find|no such).*'
 - code: -1204
-  phrase: ContainerDockerRunUnknownError
+  phrase: ContainerDockerStartUnknownError
   podPatterns:
   - containers:
     - reasonRegex: '(?i)^ContainerCannotRun$'

example/framework/README.md

@@ -9,3 +9,4 @@ We provide various Framework examples that can be submitted by various clients:
 ## Next
 1. [Framework Interop](../../doc/user-manual.md#FrameworkInterop)
 2. [Framework Usage](../../pkg/apis/frameworkcontroller/v1/types.go)
+3. [User Manual](../../doc/user-manual.md)

example/run/README.md

@@ -11,7 +11,7 @@ Notes:
  [Config Usage](../../pkg/apis/frameworkcontroller/v1/config.go) and [Config Example](../../example/config/default/frameworkcontroller.yaml).
 
 ## <a name="RunByKubernetesStatefulSet">Run By Kubernetes StatefulSet</a>
-- This approach is better for production, since StatefulSet by itself provides [self-healing](https://kubernetes.io/docs/concepts/workloads/pods/pod/#durability-of-pods-or-lack-thereof) and can ensure [at most one instance](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/storage/pod-safety.md) of FrameworkController is running at any point in time.
+- This approach is better for production, since StatefulSet by itself provides [self-healing](https://kubernetes.io/docs/concepts/workloads/pods/pod/#durability-of-pods-or-lack-thereof) and can ensure [at most one instance](https://github.com/kubernetes/community/blob/ee8998b156031f6b363daade51ca2d12521f4ac0/contributors/design-proposals/storage/pod-safety.md) of FrameworkController is running at any point in time.
 - Using official image to demonstrate this example.
 
 **Prerequisite**

pkg/apis/frameworkcontroller/v1/completion.go

@@ -252,26 +252,26 @@ func MatchCompletionCodeInfos(pod *core.Pod) PodMatchResult {
 func matchPodPattern(pod *core.Pod, podPattern *PodPattern) *MatchedPod {
 	matchedPod := &MatchedPod{}
 
-	if ms := podPattern.NameRegex.FindString(pod.Name); ms != nil {
-		if !podPattern.NameRegex.IsZero() {
+	if !podPattern.NameRegex.IsZero() {
+		if ms := podPattern.NameRegex.FindString(pod.Name); ms != nil {
 			matchedPod.Name = ms
+		} else {
+			return nil
 		}
-	} else {
-		return nil
 	}
-	if ms := podPattern.ReasonRegex.FindString(pod.Status.Reason); ms != nil {
-		if !podPattern.ReasonRegex.IsZero() {
+	if !podPattern.ReasonRegex.IsZero() {
+		if ms := podPattern.ReasonRegex.FindString(pod.Status.Reason); ms != nil {
 			matchedPod.Reason = *ms
+		} else {
+			return nil
 		}
-	} else {
-		return nil
 	}
-	if ms := podPattern.MessageRegex.FindString(pod.Status.Message); ms != nil {
-		if !podPattern.MessageRegex.IsZero() {
+	if !podPattern.MessageRegex.IsZero() {
+		if ms := podPattern.MessageRegex.FindString(pod.Status.Message); ms != nil {
 			matchedPod.Message = *ms
+		} else {
+			return nil
 		}
-	} else {
-		return nil
 	}
 
 	if len(podPattern.Containers) > 0 {
@@ -310,41 +310,53 @@ func matchContainerPattern(
 	term := container.State.Terminated
 	matchedContainer := &MatchedContainer{}
 
-	if ms := containerPattern.NameRegex.FindString(container.Name); ms != nil {
-		if !containerPattern.NameRegex.IsZero() {
+	if !containerPattern.NameRegex.IsZero() {
+		if ms := containerPattern.NameRegex.FindString(container.Name); ms != nil {
 			matchedContainer.Name = ms
+		} else {
+			return nil
 		}
-	} else {
-		return nil
 	}
-	if ms := containerPattern.ReasonRegex.FindString(term.Reason); ms != nil {
-		if !containerPattern.ReasonRegex.IsZero() {
+
+	if !containerPattern.ReasonRegex.IsZero() {
+		if term == nil {
+			return nil
+		}
+		if ms := containerPattern.ReasonRegex.FindString(term.Reason); ms != nil {
 			matchedContainer.Reason = *ms
+		} else {
+			return nil
 		}
-	} else {
-		return nil
 	}
-	if ms := containerPattern.MessageRegex.FindString(term.Message); ms != nil {
-		if !containerPattern.MessageRegex.IsZero() {
+	if !containerPattern.MessageRegex.IsZero() {
+		if term == nil {
+			return nil
+		}
+		if ms := containerPattern.MessageRegex.FindString(term.Message); ms != nil {
 			matchedContainer.Message = *ms
+		} else {
+			return nil
 		}
-	} else {
-		return nil
 	}
-
-	if containerPattern.SignalRange.Contains(term.Signal) {
-		if !containerPattern.SignalRange.IsZero() {
+	if !containerPattern.SignalRange.IsZero() {
+		if term == nil {
+			return nil
+		}
+		if containerPattern.SignalRange.Contains(term.Signal) {
 			matchedContainer.Signal = term.Signal
+		} else {
+			return nil
 		}
-	} else {
-		return nil
 	}
-	if containerPattern.CodeRange.Contains(term.ExitCode) {
-		if !containerPattern.CodeRange.IsZero() {
+	if !containerPattern.CodeRange.IsZero() {
+		if term == nil {
+			return nil
+		}
+		if containerPattern.CodeRange.Contains(term.ExitCode) {
 			matchedContainer.Code = &term.ExitCode
+		} else {
+			return nil
 		}
-	} else {
-		return nil
 	}
 
 	return matchedContainer
@@ -530,16 +542,17 @@ func ExtractPodCompletionStatus(pod *core.Pod) *PodCompletionStatus {
 	}
 
 	for _, container := range GetAllContainerStatuses(pod) {
+		ccs := &ContainerCompletionStatus{
+			Name: container.Name,
+		}
 		term := container.State.Terminated
-		pcs.Containers = append(pcs.Containers,
-			&ContainerCompletionStatus{
-				Name:    container.Name,
-				Reason:  term.Reason,
-				Message: term.Message,
-				Signal:  term.Signal,
-				Code:    term.ExitCode,
-			},
-		)
+		if term != nil {
+			ccs.Reason = term.Reason
+			ccs.Message = term.Message
+			ccs.Signal = term.Signal
+			ccs.Code = term.ExitCode
+		}
+		pcs.Containers = append(pcs.Containers, ccs)
 	}
 
 	return pcs