Skip to content

Commit 8cf8718

Browse files
Rizwana777jgwest
Rizwana777
and
jgwest
authored
fix: e2e failures in cluster mode (#998)
Signed-off-by: Rizwana777 <rizwananaaz177@gmail.com> Signed-off-by: Jonathan West <jgwest@gmail.com> Co-authored-by: Jonathan West <jgwest@gmail.com>
1 parent 073fd8c commit 8cf8718

16 files changed

Lines changed: 290 additions & 158 deletions

‎hack/dev-env/create-agent-config.sh‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,12 @@ if test "${ARGOCD_AGENT_IN_CLUSTER}" = ""; then
3838
ARGOCD_AGENT_RESOURCE_PROXY_SAN="--ip 127.0.0.1,${IPADDR}"
3939
else
4040
ARGOCD_AGENT_GRPC_SVC=$(getExternalLoadBalancerIP ${ARGOCD_AGENT_PRINCIPAL_CONTEXT} ${ARGOCD_PRINCIPAL_NAMESPACE} argocd-agent-principal)
41-
ARGOCD_AGENT_GRPC_SAN="--ip 127.0.0.1,$ARGOCD_AGENT_GRPC_SVC"
41+
# OpenShift/AWS LoadBalancers often expose a hostname instead of an IP.
42+
if [[ "$ARGOCD_AGENT_GRPC_SVC" =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}$ ]]; then
43+
ARGOCD_AGENT_GRPC_SAN="--ip 127.0.0.1,${ARGOCD_AGENT_GRPC_SVC}"
44+
else
45+
ARGOCD_AGENT_GRPC_SAN="--ip 127.0.0.1 --dns ${ARGOCD_AGENT_GRPC_SVC}"
46+
fi
4247
ARGOCD_AGENT_RESOURCE_PROXY=argocd-agent-resource-proxy
4348
ARGOCD_AGENT_RESOURCE_PROXY_SAN="--dns ${ARGOCD_AGENT_RESOURCE_PROXY}"
4449
fi

‎hack/dev-env/deploy.sh‎

Lines changed: 92 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,9 @@ cleanup() {
5151
trap cleanup EXIT
5252

5353
cp -a ${BASEPATH}/install/kubernetes/* ${TMPDIR}
54+
cp -a ${BASEPATH}/install/kubernetes/agent ${TMPDIR}/agent-managed
55+
cp -a ${BASEPATH}/install/kubernetes/agent ${TMPDIR}/agent-autonomous
56+
rm -rf ${TMPDIR}/agent
5457

5558
deploy_principal() {
5659
(
@@ -59,9 +62,12 @@ deploy_principal() {
5962
kustomize edit set image argocd-agent=${IMAGE_NAME}
6063
)
6164
sed -i'' \
65+
-e "s/ principal.listen.host:.*/ principal.listen.host: \"0.0.0.0\"/" \
6266
-e "s/ principal.namespace:.*/ principal.namespace: \"${ARGOCD_PRINCIPAL_NAMESPACE}\"/" \
6367
-e "s/ principal.allowed-namespaces:.*/ principal.allowed-namespaces: \"agent-*\"/" \
6468
principal-params-cm.yaml
69+
set_rbac_subject_namespace ${ARGOCD_PRINCIPAL_NAMESPACE} \
70+
principal-rolebinding.yaml principal-clusterrolebinding.yaml
6571
kustomize build . | kubectl --context ${ARGOCD_AGENT_PRINCIPAL_CONTEXT} -n ${ARGOCD_PRINCIPAL_NAMESPACE} apply -f -
6672
kubectl --context ${ARGOCD_AGENT_PRINCIPAL_CONTEXT} -n ${ARGOCD_PRINCIPAL_NAMESPACE} rollout restart deployment argocd-agent-principal
6773
)
@@ -70,33 +76,115 @@ deploy_principal() {
7076
deploy_agent_managed() {
7177
(
7278
principal_addr=$(getExternalLoadBalancerIP ${ARGOCD_AGENT_PRINCIPAL_CONTEXT} ${ARGOCD_PRINCIPAL_NAMESPACE} argocd-agent-principal)
73-
cd ${TMPDIR}/agent && (
79+
cd ${TMPDIR}/agent-managed && (
7480
kustomize edit set namespace ${ARGOCD_MANAGED_NAMESPACE}
7581
kustomize edit set image argocd-agent=${IMAGE_NAME}
7682
)
83+
# Modify the ClusterRole to include RBAC required by E2E tests (which are written for the configuration where agent runs locally via goreman)
84+
cat >> agent-clusterrole.yaml <<-'EOF'
85+
- apiGroups:
86+
- ""
87+
resources:
88+
- configmaps
89+
verbs:
90+
- create
91+
- delete
92+
- get
93+
- list
94+
- update
95+
- watch
96+
- patch
97+
- apiGroups:
98+
- ""
99+
resources:
100+
- pods
101+
- pods/log
102+
- pods/exec
103+
verbs:
104+
- create
105+
- get
106+
- list
107+
- watch
108+
- apiGroups:
109+
- apps
110+
resources:
111+
- deployments
112+
- replicasets
113+
verbs:
114+
- create
115+
- update
116+
- delete
117+
- get
118+
- list
119+
- watch
120+
- patch
121+
EOF
77122
sed -i'' \
78123
-e "s/ agent.namespace:.*/ agent.namespace: \"${ARGOCD_MANAGED_NAMESPACE}\"/" \
79124
-e "s/ agent.mode:.*/ agent.mode: \"managed\"/" \
80125
-e "s/ agent.creds:.*/ agent.creds: \"mtls:any\"/" \
81126
-e "s/ agent.server.address:.*/ agent.server.address: \"$principal_addr\"/" \
82127
agent-params-cm.yaml
128+
set_rbac_subject_namespace ${ARGOCD_MANAGED_NAMESPACE} \
129+
agent-rolebinding.yaml agent-clusterrolebinding.yaml
83130
kustomize build . | kubectl --context ${ARGOCD_AGENT_MANAGED_CONTEXT} -n ${ARGOCD_MANAGED_NAMESPACE} apply -f -
84131
)
85132
}
86133

87134
deploy_agent_autonomous() {
88135
(
89136
principal_addr=$(getExternalLoadBalancerIP ${ARGOCD_AGENT_PRINCIPAL_CONTEXT} ${ARGOCD_PRINCIPAL_NAMESPACE} argocd-agent-principal)
90-
cd ${TMPDIR}/agent && (
137+
cd ${TMPDIR}/agent-autonomous && (
91138
kustomize edit set namespace ${ARGOCD_AUTONOMOUS_NAMESPACE}
92139
kustomize edit set image argocd-agent=${IMAGE_NAME}
93140
)
141+
# Modify the ClusterRole to include RBAC required by E2E tests (which are written for the configuration where agent runs locally via goreman)
142+
cat >> agent-clusterrole.yaml <<-'EOF'
143+
- apiGroups:
144+
- ""
145+
resources:
146+
- configmaps
147+
verbs:
148+
- create
149+
- delete
150+
- get
151+
- list
152+
- update
153+
- watch
154+
- patch
155+
- apiGroups:
156+
- ""
157+
resources:
158+
- pods
159+
- pods/log
160+
- pods/exec
161+
verbs:
162+
- create
163+
- get
164+
- list
165+
- watch
166+
- apiGroups:
167+
- apps
168+
resources:
169+
- deployments
170+
- replicasets
171+
verbs:
172+
- create
173+
- update
174+
- delete
175+
- get
176+
- list
177+
- watch
178+
- patch
179+
EOF
94180
sed -i'' \
95181
-e "s/ agent.namespace:.*/ agent.namespace: \"${ARGOCD_AUTONOMOUS_NAMESPACE}\"/" \
96182
-e "s/ agent.mode:.*/ agent.mode: \"autonomous\"/" \
97183
-e "s/ agent.creds:.*/ agent.creds: \"mtls:any\"/" \
98184
-e "s/ agent.server.address:.*/ agent.server.address: \"$principal_addr\"/" \
99185
agent-params-cm.yaml
186+
set_rbac_subject_namespace ${ARGOCD_AUTONOMOUS_NAMESPACE} \
187+
agent-rolebinding.yaml agent-clusterrolebinding.yaml
100188
kustomize build . | kubectl --context ${ARGOCD_AGENT_AUTONOMOUS_CONTEXT} -n ${ARGOCD_AUTONOMOUS_NAMESPACE} apply -f -
101189
)
102190
}
@@ -110,14 +198,14 @@ undeploy_principal() {
110198

111199
undeploy_agent_managed() {
112200
(
113-
cd ${TMPDIR}/agent && kustomize edit set namespace ${ARGOCD_MANAGED_NAMESPACE}
201+
cd ${TMPDIR}/agent-managed && kustomize edit set namespace ${ARGOCD_MANAGED_NAMESPACE}
114202
kustomize build . | kubectl --context ${ARGOCD_AGENT_MANAGED_CONTEXT} -n ${ARGOCD_MANAGED_NAMESPACE} delete -f -
115203
)
116204
}
117205

118206
undeploy_agent_autonomous() {
119207
(
120-
cd ${TMPDIR}/agent && kustomize edit set namespace ${ARGOCD_AUTONOMOUS_NAMESPACE}
208+
cd ${TMPDIR}/agent-autonomous && kustomize edit set namespace ${ARGOCD_AUTONOMOUS_NAMESPACE}
121209
kustomize build . | kubectl --context ${ARGOCD_AGENT_AUTONOMOUS_CONTEXT} -n ${ARGOCD_AUTONOMOUS_NAMESPACE} delete -f -
122210
)
123211
}

‎hack/dev-env/utility.sh‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,15 @@
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
1414

15+
# set_rbac_subject_namespace rewrites the ServiceAccount subject namespace in
16+
# RoleBinding/ClusterRoleBinding manifests. Required when deploying to namespaces
17+
# other than the install default of "argocd" (e.g. argocd-principal).
18+
set_rbac_subject_namespace() {
19+
local namespace="$1"
20+
shift
21+
sed -i.bak -e "s/ namespace: argocd/ namespace: ${namespace}/" "$@"
22+
}
23+
1524
# getExternalLoadBalancerIP will set EXTERNAL_IP with the load balancer hostname from the specified Service
1625
getExternalLoadBalancerIP() {
1726
CONTEXT=$1

‎test/e2e/adoption_test.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -40,15 +40,15 @@ func (suite *AdoptionTestSuite) SetupTest() {
4040
suite.BaseSuite.SetupTest()
4141

4242
// Ensure principal and managed agent are running
43-
if !fixture.IsProcessRunning(fixture.PrincipalName) {
44-
suite.Require().NoError(fixture.StartProcess(fixture.PrincipalName))
43+
if !fixture.IsProcessRunning(fixture.PrincipalName, suite.T()) {
44+
suite.Require().NoError(fixture.StartProcess(fixture.PrincipalName, suite.T()))
4545
fixture.CheckReadiness(suite.T(), fixture.PrincipalName)
4646
} else {
4747
fixture.CheckReadiness(suite.T(), fixture.PrincipalName)
4848
}
4949

50-
if !fixture.IsProcessRunning(fixture.AgentManagedName) {
51-
suite.Require().NoError(fixture.StartProcess(fixture.AgentManagedName))
50+
if !fixture.IsProcessRunning(fixture.AgentManagedName, suite.T()) {
51+
suite.Require().NoError(fixture.StartProcess(fixture.AgentManagedName, suite.T()))
5252
fixture.CheckReadiness(suite.T(), fixture.AgentManagedName)
5353
} else {
5454
fixture.CheckReadiness(suite.T(), fixture.AgentManagedName)

‎test/e2e/appcache_test.go‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -43,18 +43,18 @@ func (suite *CacheTestSuite) TearDownTest() {
4343
}
4444

4545
func (suite *CacheTestSuite) SetupTest() {
46-
if !fixture.IsProcessRunning(fixture.PrincipalName) {
46+
if !fixture.IsProcessRunning(fixture.PrincipalName, suite.T()) {
4747
// Start the principal if it is not running and wait for it to be ready
48-
suite.Require().NoError(fixture.StartProcess(fixture.PrincipalName))
48+
suite.Require().NoError(fixture.StartProcess(fixture.PrincipalName, suite.T()))
4949
fixture.CheckReadiness(suite.T(), fixture.PrincipalName)
5050
} else {
5151
// If principal is already running, verify that it is ready
5252
fixture.CheckReadiness(suite.T(), fixture.PrincipalName)
5353
}
5454

55-
if !fixture.IsProcessRunning(fixture.AgentManagedName) {
55+
if !fixture.IsProcessRunning(fixture.AgentManagedName, suite.T()) {
5656
// Start the agent if it is not running and wait for it to be ready
57-
suite.Require().NoError(fixture.StartProcess(fixture.AgentManagedName))
57+
suite.Require().NoError(fixture.StartProcess(fixture.AgentManagedName, suite.T()))
5858
fixture.CheckReadiness(suite.T(), fixture.AgentManagedName)
5959
} else {
6060
// If agent is already running, verify that it is ready
@@ -108,16 +108,16 @@ func (suite *CacheTestSuite) Test_RevertDisconnectedManagedClusterChanges() {
108108

109109
// Case 1: Agent is disconnected with principal, now modify the application directly in the managed-cluster,
110110
// but changes should be reverted, to be in sync with last known state of principal application
111-
requires.NoError(fixture.StopProcess(fixture.PrincipalName))
111+
requires.NoError(fixture.StopProcess(fixture.PrincipalName, suite.T()))
112112
requires.Eventually(func() bool {
113-
return !fixture.IsProcessRunning(fixture.PrincipalName)
113+
return !fixture.IsProcessRunning(fixture.PrincipalName, suite.T())
114114
}, 30*time.Second, 1*time.Second)
115115

116116
updateAppInfo(suite.Ctx, suite.ManagedAgentClient, agentKey, []string{"a", "b"}, requires)
117117
validateAppReverted(suite.Ctx, suite.ManagedAgentClient, &app, agentKey, requires, suite.T())
118118

119119
// Case 2: Agent is reconnected with principal and now changes done in principal should reflect in managed-cluster
120-
requires.NoError(fixture.StartProcess(fixture.PrincipalName))
120+
requires.NoError(fixture.StartProcess(fixture.PrincipalName, suite.T()))
121121
fixture.CheckReadiness(suite.T(), fixture.PrincipalName)
122122

123123
updateAppInfo(suite.Ctx, suite.PrincipalClient, principalKey, []string{"a", "b"}, requires)
@@ -164,17 +164,17 @@ func (suite *CacheTestSuite) Test_RevertManagedClusterOfflineChanges() {
164164
// Agent in not running, but still make changes in the managed-cluster application manifest.
165165
// When agent is restarted, these changes should be reverted to be in sync with principal.
166166
requires.Eventually(func() bool {
167-
return fixture.IsProcessRunning(fixture.AgentManagedName)
167+
return fixture.IsProcessRunning(fixture.AgentManagedName, suite.T())
168168
}, 60*time.Second, 1*time.Second)
169169

170-
requires.NoError(fixture.StopProcess(fixture.AgentManagedName))
170+
requires.NoError(fixture.StopProcess(fixture.AgentManagedName, suite.T()))
171171
requires.Eventually(func() bool {
172-
return !fixture.IsProcessRunning(fixture.AgentManagedName)
172+
return !fixture.IsProcessRunning(fixture.AgentManagedName, suite.T())
173173
}, 60*time.Second, 1*time.Second)
174174

175175
updateAppInfo(suite.Ctx, suite.ManagedAgentClient, agentKey, []string{"a", "b"}, requires)
176176

177-
requires.NoError(fixture.StartProcess(fixture.AgentManagedName))
177+
requires.NoError(fixture.StartProcess(fixture.AgentManagedName, suite.T()))
178178
fixture.CheckReadiness(suite.T(), fixture.AgentManagedName)
179179
validateAppReverted(suite.Ctx, suite.ManagedAgentClient, &app, agentKey, requires, suite.T())
180180
}

‎test/e2e/clusterinfo_test.go‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -41,27 +41,27 @@ func TestClusterTestSuite(t *testing.T) {
4141
}
4242

4343
func (suite *ClusterInfoTestSuite) SetupTest() {
44-
if !fixture.IsProcessRunning(fixture.PrincipalName) {
44+
if !fixture.IsProcessRunning(fixture.PrincipalName, suite.T()) {
4545
// Start the principal if it is not running and wait for it to be ready
46-
suite.Require().NoError(fixture.StartProcess(fixture.PrincipalName))
46+
suite.Require().NoError(fixture.StartProcess(fixture.PrincipalName, suite.T()))
4747
fixture.CheckReadiness(suite.T(), fixture.PrincipalName)
4848
} else {
4949
// If principal is already running, verify that it is ready
5050
fixture.CheckReadiness(suite.T(), fixture.PrincipalName)
5151
}
5252

53-
if !fixture.IsProcessRunning(fixture.AgentManagedName) {
53+
if !fixture.IsProcessRunning(fixture.AgentManagedName, suite.T()) {
5454
// Start the agent if it is not running and wait for it to be ready
55-
suite.Require().NoError(fixture.StartProcess(fixture.AgentManagedName))
55+
suite.Require().NoError(fixture.StartProcess(fixture.AgentManagedName, suite.T()))
5656
fixture.CheckReadiness(suite.T(), fixture.AgentManagedName)
5757
} else {
5858
// If agent is already running, verify that it is ready
5959
fixture.CheckReadiness(suite.T(), fixture.AgentManagedName)
6060
}
6161

62-
if !fixture.IsProcessRunning(fixture.AgentAutonomousName) {
62+
if !fixture.IsProcessRunning(fixture.AgentAutonomousName, suite.T()) {
6363
// Start the agent if it is not running and wait for it to be ready
64-
suite.Require().NoError(fixture.StartProcess(fixture.AgentAutonomousName))
64+
suite.Require().NoError(fixture.StartProcess(fixture.AgentAutonomousName, suite.T()))
6565
fixture.CheckReadiness(suite.T(), fixture.AgentAutonomousName)
6666
} else {
6767
// If agent is already running, verify that it is ready
@@ -87,7 +87,7 @@ func (suite *ClusterInfoTestSuite) Test_ClusterInfo_Managed() {
8787
}, 60*time.Second, 1*time.Second)
8888

8989
// Stop the agent
90-
err := fixture.StopProcess(fixture.AgentManagedName)
90+
err := fixture.StopProcess(fixture.AgentManagedName, suite.T())
9191
requires.NoError(err)
9292

9393
// Verify that connection status is updated when agent is disconnected
@@ -100,7 +100,7 @@ func (suite *ClusterInfoTestSuite) Test_ClusterInfo_Managed() {
100100
}, 60*time.Second, 1*time.Second)
101101

102102
// Restart the agent
103-
err = fixture.StartProcess(fixture.AgentManagedName)
103+
err = fixture.StartProcess(fixture.AgentManagedName, suite.T())
104104
requires.NoError(err)
105105
fixture.CheckReadiness(suite.T(), fixture.AgentManagedName)
106106

@@ -127,7 +127,7 @@ func (suite *ClusterInfoTestSuite) Test_ClusterInfo_Autonomous() {
127127
}, 60*time.Second, 1*time.Second)
128128

129129
// Stop the agent
130-
err := fixture.StopProcess(fixture.AgentAutonomousName)
130+
err := fixture.StopProcess(fixture.AgentAutonomousName, suite.T())
131131
requires.NoError(err)
132132

133133
// Verify that connection status is updated when agent is disconnected
@@ -140,7 +140,7 @@ func (suite *ClusterInfoTestSuite) Test_ClusterInfo_Autonomous() {
140140
}, 60*time.Second, 1*time.Second)
141141

142142
// Restart the agent
143-
err = fixture.StartProcess(fixture.AgentAutonomousName)
143+
err = fixture.StartProcess(fixture.AgentAutonomousName, suite.T())
144144
requires.NoError(err)
145145
fixture.CheckReadiness(suite.T(), fixture.AgentAutonomousName)
146146

@@ -231,7 +231,7 @@ func (suite *ClusterInfoTestSuite) Test_ClusterCacheInfo() {
231231

232232
// Step 8:
233233
// Disconnect agent and verify that connection status is changed to Failed
234-
requires.NoError(fixture.StopProcess(fixture.AgentManagedName))
234+
requires.NoError(fixture.StopProcess(fixture.AgentManagedName, suite.T()))
235235
requires.Eventually(func() bool {
236236
return fixture.HasConnectionStatus(fixture.AgentManagedName, appv1.ConnectionState{
237237
Status: appv1.ConnectionStatusFailed,
@@ -252,7 +252,7 @@ func (suite *ClusterInfoTestSuite) Test_ClusterCacheInfo() {
252252

253253
// Step 10:
254254
// Reconnect agent and verify that connection status and cluster cache info are updated again with correct values
255-
requires.NoError(fixture.StartProcess(fixture.AgentManagedName))
255+
requires.NoError(fixture.StartProcess(fixture.AgentManagedName, suite.T()))
256256
fixture.CheckReadiness(suite.T(), fixture.AgentManagedName)
257257

258258
requires.Eventually(func() bool {

‎test/e2e/fixture/fixture.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import (
1919
"fmt"
2020
"os"
2121
"reflect"
22+
"testing"
2223
"time"
2324

2425
"github.com/argoproj-labs/argocd-agent/internal/manager"
@@ -522,3 +523,11 @@ func resetManagedAgentClusterInfo(clusterDetails *ClusterDetails) error {
522523
}
523524
return nil
524525
}
526+
527+
// SkipIfAgentInClusterEnvVarIsSet should be called to verify that the test (or utility function) is running locally (that is, not on cluster). If the test is running on cluster, the correct step is to skip the test.
528+
func SkipIfAgentInClusterEnvVarIsSet(t *testing.T) {
529+
t.Helper()
530+
if os.Getenv("ARGOCD_AGENT_IN_CLUSTER") == "true" {
531+
t.Skip("skipping because ARGOCD_AGENT_IN_CLUSTER=true")
532+
}
533+
}

0 commit comments

Comments
 (0)