eve-k: reduce idle logging, ignore api outage during failover

The 'looking for cont' log near the top of checkAppsStatus
and checkAppsFailover is run when apps are running outside of
any active management operations or app migration operations.
Avoid logging above Functionf in this state to minimize pillar logs.

Kubevirt operations on the Hypervisor interface which either change
the state of a domain or get state of a domain specific to a host/node
should first call scheduledOnNode which implements vm/container handlers.
Don't call the direct replicaVmiScheduledOnMe or replicaPodScheduledOnMe.

During failover multiple nodes may attempt info on a kubevirt domain.
scheduledOnNode is used to skip operations for nodes where an app is not
running. In cases where scheduledOnMe is unable to determine the scheduling
state of an app: the caller should determine if the error is some type
where the k3s api server may be temporarily unavailable.
If the api server is temporarily unavailable then declare app state UNKNOWN
as BROKEN signals to the controller that no further tries will be attempted.

Remove incorrect error log in getNodeDrainRequester(). Its possible to
meet that error log in normal conditions:
- zedkube startup as single node mode
- zedkube recv edgenodeclusterconfig
- zedkube node delete
- zedkube cluster delete

Signed-off-by: Andrew Durbin <andrewd@zededa.com>

Author: andrewd-zededa

pkg/pillar/cmd/zedkube/applogs.go

@@ -160,7 +160,7 @@ func (z *zedkube) checkAppsStatus() {
 		// Both Pods will be of the pattern <appname>-<uuid prefix>-<pod uuid suffix>
 		for _, pod := range pods.Items {
 			contVMIName := "virt-launcher-" + contName
-			log.Noticef("checkAppsStatus: pod %s, looking for cont %s", pod.Name, contName)
+			log.Functionf("checkAppsStatus: pod %s, looking for cont %s", pod.Name, contName)
 			foundVMIPod := strings.HasPrefix(pod.Name, contVMIName)
 			if strings.HasPrefix(pod.Name, contName) || foundVMIPod {
 				// Case 1

pkg/pillar/cmd/zedkube/failover.go

@@ -76,7 +76,7 @@ func (z *zedkube) checkAppsFailover(wdFunc func()) {
 
 		for _, pod := range pods.Items {
 			contVMIName := "virt-launcher-" + contName
-			log.Noticef("checkAppsStatus: pod %s, looking for cont %s", pod.Name, contName)
+			log.Functionf("checkAppsFailover: pod %s, looking for cont %s", pod.Name, contName)
 			foundVMIPod := strings.HasPrefix(pod.Name, contVMIName)
 			if strings.HasPrefix(pod.Name, contName) || foundVMIPod {
 				// Case 1

pkg/pillar/cmd/zedkube/handlenodedrain.go

@@ -38,7 +38,6 @@ func getNodeDrainRequester(ctx *zedkube) kubeapi.DrainRequester {
 	if len(items) == 1 {
 		return kubeapi.UPDATE
 	}
-	log.Errorf("getNodeDrainRequester should never get here")
 	return kubeapi.NONE
 }
 

pkg/pillar/hypervisor/kubevirt.go

@@ -736,7 +736,7 @@ func (ctx kubevirtContext) Delete(domainName string) (result error) {
 		return logError("delete domain %s failed to get vmlist", domainName)
 	}
 
-	onMe, err := ctx.replicaVmiScheduledOnMe(vmis.name)
+	onMe, err := ctx.scheduledOnMe(vmis.mtype, vmis.name)
 	if err != nil {
 		return err
 	}
@@ -813,9 +813,12 @@ func (ctx kubevirtContext) Info(domainName string) (int, types.SwState, error) {
 		return 0, types.HALTED, logError("info domain %s failed to get vmlist", domainName)
 	}
 
-	onMe, err := ctx.replicaVmiScheduledOnMe(vmis.name)
+	onMe, err := ctx.scheduledOnMe(vmis.mtype, vmis.name)
 	if err != nil {
-		return 0, types.BROKEN, logError("Failed to determine scheduled node")
+		if isK3sUnreachable(err) {
+			return 0, types.UNKNOWN, nil
+		}
+		return 0, types.BROKEN, logError("Failed to determine scheduled node: %s", err)
 	}
 	if !onMe {
 		return 0, types.UNKNOWN, nil