Skip to content

fix: progressive sync for applicationSet needs to evaluate more application status fields #126

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Apr 28, 2025
Merged

fix: progressive sync for applicationSet needs to evaluate more application status fields #126

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions deploy/crds/apps.open-cluster-management.io_multiclusterapplicationsetreports.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,12 @@ spec:
type: string
syncStatus:
type: string
operationStateStartedAt:
type: string
operationStatePhase:
type: string
syncRevision:
type: string
type: object
type: array
resources:
Expand Down
27 changes: 23 additions & 4 deletions e2e/run_e2e.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,8 @@ kubectl apply -f deploy/crds/
kubectl apply -f hack/test/crds/0000_00_authentication.open-cluster-management.io_managedserviceaccounts.yaml
kubectl apply -f deploy/controller/

sleep 120
kubectl -n open-cluster-management rollout status deployment multicloud-integrations-gitops --timeout=120s
kubectl -n open-cluster-management rollout status deployment multicloud-integrations --timeout=120s

echo "TEST Propgation controller startup (expecting error)"
POD_NAME=$(kubectl -n open-cluster-management get deploy multicloud-integrations -o yaml | grep ReplicaSet | grep successful | cut -d'"' -f2)
Expand All @@ -28,7 +29,11 @@ fi
echo "SETUP install Argo CD to Managed cluster"
kubectl config use-context kind-cluster1
kubectl create namespace argocd
kubectl apply -n argocd --force -f hack/test/e2e/argo-cd-install.yaml
kubectl apply -n argocd --force -f hack/test/e2e/argo-cd-install.yaml
kubectl -n argocd scale deployment/argocd-applicationset-controller --replicas 0
kubectl -n argocd scale deployment/argocd-server --replicas 0
kubectl -n argocd scale deployment/argocd-dex-server --replicas 0
kubectl -n argocd scale deployment/argocd-notifications-controller --replicas 0

echo "SETUP install Argo CD to Hub cluster"
kubectl config use-context kind-hub
Expand All @@ -44,8 +49,7 @@ kubectl -n argocd scale statefulset/argocd-application-controller --replicas 0
# enable progressive sync
kubectl -n argocd patch configmap argocd-cmd-params-cm --type merge -p '{"data":{"applicationsetcontroller.enable.progressive.syncs":"true"}}'
kubectl -n argocd rollout restart deployment argocd-applicationset-controller

sleep 60s
kubectl -n argocd rollout status deployment argocd-applicationset-controller --timeout=60s

echo "TEST Propgation controller startup"
if kubectl -n open-cluster-management logs $POD_NAME argocd-pull-integration-controller-manager | grep "Starting Controller" | grep "Application"; then
Expand Down Expand Up @@ -161,3 +165,18 @@ else
echo "Propagation FAILED: guestbook-ui deploy not created"
exit 1
fi
kubectl config use-context kind-hub
if [[ -n $(kubectl -n argocd get app cluster1-guestbook-app -o jsonpath='{.status.operationState.phase}') ]]; then
echo "Propagation: hub cluster application cluster1-guestbook-app phase is not empty"
else
echo "Propagation FAILED: hub cluster application cluster1-guestbook-app phase is empty"
kubectl -n argocd get app cluster1-guestbook-app -o yaml
exit 1
fi
if [[ -n $(kubectl -n argocd get app cluster1-guestbook-app -o jsonpath='{.status.sync.revision}') ]]; then
echo "Propagation: hub cluster application cluster1-guestbook-app revision not empty"
else
echo "Propagation FAILED: hub cluster application cluster1-guestbook-app revision is empty"
kubectl -n argocd get app cluster1-guestbook-app -o yaml
exit 1
fi
53 changes: 39 additions & 14 deletions hack/crds/apps.open-cluster-management.io_multiclusterapplicationsetreports.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
controller-gen.kubebuilder.io/version: v0.10.0
controller-gen.kubebuilder.io/version: v0.6.0
creationTimestamp: null
name: multiclusterapplicationsetreports.apps.open-cluster-management.io
spec:
Expand All @@ -21,8 +21,12 @@ spec:
- name: v1alpha1
schema:
openAPIV3Schema:
description: MulticlusterApplicationSetReport is the Schema for the MulticlusterApplicationSetReport
API.
description: MulticlusterApplicationSetReport provides a report of the status
of an application from all the managed clusters where the application is
deployed on. It provides a summary of the number of clusters in the various
states. If an error or warning occurred when installing the application
on a managed cluster, the conditions, including the waring and error message,
is captured in the report.
properties:
apiVersion:
description: 'APIVersion defines the versioned schema of this representation
Expand All @@ -38,36 +42,47 @@ spec:
type: object
statuses:
description: AppConditions defines all the error/warning conditions in
all clusters per application
all clusters where a particular application is deployed.
properties:
clusterConditions:
items:
description: ClusterCondition defines all the error/warning conditions
in one cluster per application
in one cluster for an application.
properties:
app:
type: string
cluster:
type: string
conditions:
items:
description: Condition defines a type of error/warning
properties:
message:
description: Message is the warning/error message associated
with this condition.
type: string
type:
description: Type identifies if the condition is a warning
or an error.
type: string
type: object
type: array
healthStatus:
type: string
syncStatus:
type: string
app:
operationStateStartedAt:
type: string
operationStatePhase:
type: string
syncRevision:
type: string
type: object
type: array
resources:
items:
description: ResourceRef defines a kind of resource
description: ResourceRef identifies the resource that is deployed
by the application.
properties:
apiVersion:
type: string
Expand All @@ -80,27 +95,31 @@ spec:
type: object
type: array
summary:
description: Summary provides a summary of results
description: 'ReportSummary provides a summary by providing a count
of the total number of clusters where the application is deployed.
It also provides a count of how many clusters where an application
are in the following states: synced, notSynced, healthy, notHealthy,
and inProgress.'
properties:
clusters:
description: Clusters provides the count of all managed clusters
the application is deployed to
the application is deployed.
type: string
healthy:
description: Healthy provides the count of healthy applications
description: Healthy provides the count of healthy applications.
type: string
inProgress:
description: InProgress provides the count of applications that
are in the process of being deployed
are in the process of being deployed.
type: string
notHealthy:
description: NotHealthy provides the count of non-healthy applications
description: NotHealthy provides the count of non-healthy applications.
type: string
notSynced:
description: NotSynced provides the count of the out of sync applications
description: NotSynced provides the count of the out of sync applications.
type: string
synced:
description: Synced provides the count of synced applications
description: Synced provides the count of synced applications.
type: string
type: object
type: object
Expand All @@ -109,3 +128,9 @@ spec:
type: object
served: true
storage: true
status:
acceptedNames:
kind: ""
plural: ""
conditions: []
storedVersions: []
13 changes: 8 additions & 5 deletions pkg/apis/appsetreport/v1alpha1/appsetreport_types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,11 +59,14 @@ type Condition struct {

// ClusterCondition defines all the error/warning conditions in one cluster for an application.
type ClusterCondition struct {
Cluster string `json:"cluster,omitempty"`
SyncStatus string `json:"syncStatus,omitempty"`
HealthStatus string `json:"healthStatus,omitempty"`
App string `json:"app,omitempty"`
Conditions []Condition `json:"conditions,omitempty"`
Cluster string `json:"cluster,omitempty"`
SyncStatus string `json:"syncStatus,omitempty"`
HealthStatus string `json:"healthStatus,omitempty"`
OperationStateStartedAt string `json:"operationStateStartedAt,omitempty"`
OperationStatePhase string `json:"operationStatePhase,omitempty"`
SyncRevision string `json:"syncRevision,omitempty"`
App string `json:"app,omitempty"`
Conditions []Condition `json:"conditions,omitempty"`
}

// AppConditions defines all the error/warning conditions in all clusters where a particular application is deployed.
Expand Down
36 changes: 27 additions & 9 deletions pkg/controller/multiclusterstatusaggregation/multiclusterStatusAggregation_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,9 +62,12 @@ type Cluster struct {

// Value for the appSetClusterStatusMap
type OverallStatus struct {
HealthStatus string
SyncStatus string
App string
HealthStatus string
SyncStatus string
OperationStateStartedAt string
OperationStatePhase string
SyncRevision string
App string
}

// AppSetClusterResourceSorter sorts appsetreport resources by name
Expand Down Expand Up @@ -187,6 +190,9 @@ func (r *ReconcilePullModelAggregation) generateAggregation() error {

for _, manifestWork := range appSetClusterList.Items {
healthStatus, syncStatus := "Unknown", "Unknown"
operationStatePhase := ""
operationStateStartedAt := ""
syncRevision := ""

appsetNs, appsetName := ParseNamespacedName(manifestWork.Annotations[propagation.AnnotationKeyAppSet])

Expand All @@ -203,13 +209,22 @@ func (r *ReconcilePullModelAggregation) generateAggregation() error {
healthStatus = *statuses.Value.String
} else if statuses.Name == "syncStatus" {
syncStatus = *statuses.Value.String
} else if statuses.Name == "operationStatePhase" {
operationStatePhase = *statuses.Value.String
} else if statuses.Name == "operationStateStartedAt" {
operationStateStartedAt = *statuses.Value.String
} else if statuses.Name == "syncRevision" {
syncRevision = *statuses.Value.String
}
}
}

appSetClusterStatusMap[appSetKey][clusterKey] = OverallStatus{
HealthStatus: healthStatus,
SyncStatus: syncStatus,
HealthStatus: healthStatus,
SyncStatus: syncStatus,
OperationStateStartedAt: operationStateStartedAt,
OperationStatePhase: operationStatePhase,
SyncRevision: syncRevision,
App: appsetNs + "/" + manifestWork.Annotations[propagation.AnnotationKeyHubApplicationName] +
"/" + manifestWork.Namespace + "/" + manifestWork.Name,
}
Expand Down Expand Up @@ -346,10 +361,13 @@ func (r *ReconcilePullModelAggregation) generateSummary(appSetClusterStatusMap m
klog.V(1).Info("Cluster: ", cluster)
// generate the cluster condition list per this appset
appSetClusterConditionsMap[cluster.clusterName] = appsetreportV1alpha1.ClusterCondition{
Cluster: cluster.clusterName,
SyncStatus: appSetClusterStatusMap[appset][cluster].SyncStatus,
HealthStatus: appSetClusterStatusMap[appset][cluster].HealthStatus,
App: appSetClusterStatusMap[appset][cluster].App,
Cluster: cluster.clusterName,
SyncStatus: appSetClusterStatusMap[appset][cluster].SyncStatus,
HealthStatus: appSetClusterStatusMap[appset][cluster].HealthStatus,
OperationStateStartedAt: appSetClusterStatusMap[appset][cluster].OperationStateStartedAt,
OperationStatePhase: appSetClusterStatusMap[appset][cluster].OperationStatePhase,
SyncRevision: appSetClusterStatusMap[appset][cluster].SyncRevision,
App: appSetClusterStatusMap[appset][cluster].App,
}

// Calculate the summary while we're here.
Expand Down
59 changes: 54 additions & 5 deletions propagation-controller/application/application_status_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,14 @@ import (
"encoding/json"
"fmt"
"strings"
"time"

"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/util/retry"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/log"
Expand Down Expand Up @@ -116,16 +118,62 @@ func (r *ApplicationStatusReconciler) Reconcile(ctx context.Context, req ctrl.Re
return ctrl.Result{}, err
}

if cc.HealthStatus != "" {
if cc.HealthStatus == "Healthy" {
// If the health status is Healthy then simulate Progressing first then set to Healthy for ApplicationSet controller
if err := unstructured.SetNestedField(application.Object, "Progressing", "status", "health", "status"); err != nil {
log.Error(err, "unable to set healthStatus in Application status")
return ctrl.Result{}, err
}

log.Info("updating Application health status to Progressing")

if err := retry.RetryOnConflict(retry.DefaultBackoff, func() error {
return r.Client.Update(ctx, application)
}); err != nil {
log.Error(err, "unable to update Application")
return ctrl.Result{}, err
}
}

if err := unstructured.SetNestedField(newStatus, cc.HealthStatus, "health", "status"); err != nil {
log.Error(err, "unable to set health")
return ctrl.Result{}, err
}
}

if cc.SyncStatus != "" {
if err := unstructured.SetNestedField(newStatus, cc.SyncStatus, "sync", "status"); err != nil {
log.Error(err, "unable to set sync")
return ctrl.Result{}, err
}
}

if cc.HealthStatus != "" {
if err := unstructured.SetNestedField(newStatus, cc.HealthStatus, "health", "status"); err != nil {
log.Error(err, "unable to set health")
if cc.OperationStatePhase != "" {
if err := unstructured.SetNestedField(newStatus, cc.OperationStatePhase, "operationState", "phase"); err != nil {
log.Error(err, "unable to set OperationStatePhase")
return ctrl.Result{}, err
}

if cc.OperationStateStartedAt == "" {
cc.OperationStateStartedAt = time.Now().UTC().Format(time.RFC3339)
}

if err = unstructured.SetNestedField(newStatus, cc.OperationStateStartedAt, "operationState", "startedAt"); err != nil {
log.Error(err, "unable to set OperationStateStartedAt")
return ctrl.Result{}, err
}

if err = unstructured.SetNestedField(newStatus, map[string]interface{}{}, "operationState", "operation"); err != nil {
// Application required field, set it to empty object
log.Error(err, "unable to set operation")
return ctrl.Result{}, err
}
}

if cc.SyncRevision != "" {
if err := unstructured.SetNestedField(newStatus, cc.SyncRevision, "sync", "revision"); err != nil {
log.Error(err, "unable to set SyncRevision")
return ctrl.Result{}, err
}
}
Expand Down Expand Up @@ -172,8 +220,9 @@ func (r *ApplicationStatusReconciler) Reconcile(ctx context.Context, req ctrl.Re
return ctrl.Result{}, err
}

err = r.Client.Update(ctx, application)
if err != nil {
if err := retry.RetryOnConflict(retry.DefaultBackoff, func() error {
return r.Client.Update(ctx, application)
}); err != nil {
log.Error(err, "unable to update Application")
return ctrl.Result{}, err
}
Expand Down
Loading