Enable strict node scheduling for cluster app instances.

App wil only run if scheduling succeeds for the requested node id.
Set DesignatedNodeID and AffinityType Required.

With strict node scheduling the app will not failover to other cluster nodes.

Signed-off-by: Andrew Durbin <andrewd@zededa.com>

Author: andrewd-zededa

pkg/pillar/cmd/zedagent/parseconfig.go

@@ -838,6 +838,12 @@ func parseAppInstanceConfig(getconfigCtx *getconfigContext,
 		} else {
 			appInstance.IsDesignatedNodeID = true
 		}
+		switch cfgApp.GetAffinity() {
+		case zconfig.AffinityType_AFFINITY_TYPE_REQUIRED:
+			appInstance.AffinityType = types.RequiredDuringScheduling
+		default:
+			appInstance.AffinityType = types.PreferredDuringScheduling
+		}
 
 		// Verify that it fits and if not publish with error
 		checkAndPublishAppInstanceConfig(getconfigCtx.pubAppInstanceConfig, appInstance)

pkg/pillar/cmd/zedkube/failover.go

@@ -52,6 +52,13 @@ func (z *zedkube) checkAppsFailover(wdFunc func()) {
 		wdFunc()
 
 		aiconfig := item.(types.AppInstanceConfig)
+		if aiconfig.AffinityType == types.RequiredDuringScheduling {
+			// Don't attempt failover on apps with strict node affinity
+			log.Noticef("checkAppsFailover: AppInstanceConfig id:%s name:%s has affinityType:required, failover is disabled",
+				aiconfig.UUIDandVersion.UUID, aiconfig.DisplayName)
+			continue
+		}
+
 		encAppStatus := types.ENClusterAppStatus{
 			AppUUID:    aiconfig.UUIDandVersion.UUID,
 			IsDNidNode: aiconfig.IsDesignatedNodeID,

pkg/pillar/cmd/zedmanager/handledomainmgr.go

@@ -64,7 +64,8 @@ func MaybeAddDomainConfig(ctx *zedmanagerContext,
 		DisableLogs:       aiConfig.DisableLogs,
 		// This isDNiDnode will be set to true even if the App is not in cluster mode,
 		// This will be set in zedagent parseConfig for the case of single node/device App case.
-		IsDNidNode: aiConfig.IsDesignatedNodeID,
+		IsDNidNode:   aiConfig.IsDesignatedNodeID,
+		AffinityType: aiConfig.AffinityType,
 		// DeploymentType is set to the value of the DeploymentType of the AppInstanceConfig
 		DeploymentType: aiConfig.DeploymentType,
 	}

pkg/pillar/docs/failover.md

@@ -120,15 +120,23 @@ The designated device will start the application and publishes AppInstanceStatus
 
 There is also additional flag in AppInstanceStatus named NoUploadStatsToController. Zedagent looks at that flag and decides to upload stats to controller or not. The reason to have such flag is that app can move between nodes and only one node is supposed to upload stats to controller. Hence that flag will be toggled accordingly.
 
-eve-api has been enhanced to add designated node id to AppInstanceConfig
+eve-api has been enhanced to add designated node id and affinity type to AppInstanceConfig
 
 * [config/appinfo.config](https://github.com/lf-edge/eve-api/blob/main/proto/config/appconfig.proto)
 
 ```golang
   message AppInstanceConfig {
   ....
-  // This edge-node UUID for the Designate Node for the Application
+  // Designated Node Id is used for cluster nodes to determine placement of
+  // the AppInstance when an EdgeNodeCluster config is present.
+  // See affinity below to set the desired node affinity.
+  // eg. Preferred or Required.
   string designated_node_id = 26;
+  ....
+  // Affinity is used for cluster nodes to determine
+  // preferred or required node scheduling for app instances.
+  // Node Id for scheduling is defined in designated_node_id.
+  AffinityType affinity = 29;
   }
 ```
 
@@ -143,7 +151,8 @@ eve-api has been enhanced to add designated node id to AppInstanceConfig
   }
 ```
 
-EVE specific changes to AppInstanceConfig and AppInstanceStatus structs to carry bool IsDesignatedNodeID
+EVE specific changes to AppInstanceConfig and AppInstanceStatus structs to carry bool IsDesignatedNodeID.
+AffinityType is also added to AppInstanceConfig
 
 * [types/zedmanagertypes.go](pkg/pillar/types/zedmanagertypes.go)
 
@@ -152,6 +161,9 @@ EVE specific changes to AppInstanceConfig and AppInstanceStatus structs to carry
   ....
   // Am I Cluster Designated Node Id for this app
   IsDesignatedNodeID bool
+
+  // Node Affinity for cluster IsDesignatedNodeID
+  AffinityType Affinity
   }
   type AppInstanceStatus struct {
   ....
@@ -181,6 +193,8 @@ There are various scenarios that can trigger the application failover. Some of t
 
 We depend on the kubernetes infrastructure to detect and the trigger the failover of an application.
 Kubernetes scheduler makes the decision to move the app to some other existing node in a cluster.
+If an application is defined with AppInstanceConfig.AffinityType==RequiredDuringScheduling then
+the application cannot failover to another node, failover is disabled for only that application.
 
 Once the application gets Scheduled on a particular node after failover. EVE code does the following;
 

pkg/pillar/hypervisor/kubevirt.go

@@ -455,7 +455,7 @@ func (ctx kubevirtContext) CreateReplicaVMIConfig(domainName string, config type
 	}
 
 	// Set the affinity to this node the VMI is preferred to run on
-	affinity := setKubeAffinity(nodeName)
+	affinity := setKubeAffinity(nodeName, config.AffinityType)
 
 	// Set tolerations to handle node conditions
 	tolerations := setKubeToleration(int64(tolerateSec))
@@ -1337,7 +1337,7 @@ func (ctx kubevirtContext) CreateReplicaPodConfig(domainName string, config type
 				},
 				Spec: k8sv1.PodSpec{
 					Tolerations: setKubeToleration(int64(tolerateSec)),
-					Affinity:    setKubeAffinity(nodeName),
+					Affinity:    setKubeAffinity(nodeName, config.AffinityType),
 					Containers: []k8sv1.Container{
 						{
 							Name:            kubeName,
@@ -1421,26 +1421,38 @@ func (ctx kubevirtContext) CreateReplicaPodConfig(domainName string, config type
 	return nil
 }
 
-func setKubeAffinity(nodeName string) *k8sv1.Affinity {
-	affinity := &k8sv1.Affinity{
-		NodeAffinity: &k8sv1.NodeAffinity{
-			PreferredDuringSchedulingIgnoredDuringExecution: []k8sv1.PreferredSchedulingTerm{
+func setKubeAffinity(nodeName string, affinityType types.Affinity) *k8sv1.Affinity {
+	matchExpressions := []k8sv1.NodeSelectorRequirement{
+		{
+			Key:      "kubernetes.io/hostname",
+			Operator: "In",
+			Values:   []string{nodeName},
+		},
+	}
+
+	k8sAffinity := &k8sv1.Affinity{
+		NodeAffinity: &k8sv1.NodeAffinity{},
+	}
+	switch affinityType {
+	case types.PreferredDuringScheduling:
+		k8sAffinity.NodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution = []k8sv1.PreferredSchedulingTerm{
+			{
+				Preference: k8sv1.NodeSelectorTerm{
+					MatchExpressions: matchExpressions,
+				},
+				Weight: 100,
+			},
+		}
+	case types.RequiredDuringScheduling:
+		k8sAffinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution = &k8sv1.NodeSelector{
+			NodeSelectorTerms: []k8sv1.NodeSelectorTerm{
 				{
-					Preference: k8sv1.NodeSelectorTerm{
-						MatchExpressions: []k8sv1.NodeSelectorRequirement{
-							{
-								Key:      "kubernetes.io/hostname",
-								Operator: "In",
-								Values:   []string{nodeName},
-							},
-						},
-					},
-					Weight: 100,
+					MatchExpressions: matchExpressions,
 				},
 			},
-		},
+		}
 	}
-	return affinity
+	return k8sAffinity
 }
 
 func setKubeToleration(timeOutSec int64) []k8sv1.Toleration {

pkg/pillar/types/domainmgrtypes.go

@@ -46,6 +46,9 @@ type DomainConfig struct {
 	// if this node is the DNiD of the App
 	IsDNidNode bool
 
+	// Node Affinity for cluster IsDesignatedNodeID
+	AffinityType Affinity
+
 	// XXX: to be deprecated, use CipherBlockStatus instead
 	CloudInitUserData *string `json:"pubsub-large-CloudInitUserData"` // base64-encoded
 

pkg/pillar/types/zedmanagertypes.go

@@ -168,6 +168,9 @@ type AppInstanceConfig struct {
 	// Am I Cluster Designated Node Id for this app
 	IsDesignatedNodeID bool
 
+	// Node Affinity for cluster IsDesignatedNodeID
+	AffinityType Affinity
+
 	// AppRuntimeType specifies the runtime type of the application
 	DeploymentType AppRuntimeType
 
@@ -558,3 +561,15 @@ func GetSnapshotInstanceStatusFile(snapshotID string) string {
 func GetSnapshotAppInstanceConfigFile(snapshotID string) string {
 	return filepath.Join(GetSnapshotDir(snapshotID), SnapshotAppInstanceConfigFilename)
 }
+
+// Affinity - Cluster App Instance Node Affinity
+type Affinity uint8
+
+const (
+	// PreferredDuringScheduling - designated_node_id is preferred for placement but
+	// if the node is unhealthy, app can run on other nodes.
+	PreferredDuringScheduling Affinity = iota
+	// RequiredDuringScheduling - designated_node_id is required to be healthy for
+	// app instance to boot, app will not failover to other nodes.
+	RequiredDuringScheduling
+)