Skip to content

Commit 100bbc8

Browse files
dspodspo
authored
chore: Add GatewayClass protection and deletion logic (#117)
1 parent 356694f commit 100bbc8

File tree

7 files changed

+119
-11
lines changed

7 files changed

+119
-11
lines changed

‎internal/controller/gatewayclass_congroller.go

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,18 +3,26 @@ package controller
33
import (
44
"context"
55
"fmt"
6+
"time"
67

78
"github.com/go-logr/logr"
89
meta "k8s.io/apimachinery/pkg/apis/meta/v1"
910
"k8s.io/apimachinery/pkg/runtime"
11+
"k8s.io/apimachinery/pkg/types"
12+
"k8s.io/client-go/tools/record"
1013
ctrl "sigs.k8s.io/controller-runtime"
1114
"sigs.k8s.io/controller-runtime/pkg/client"
15+
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
1216
"sigs.k8s.io/controller-runtime/pkg/predicate"
1317
gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"
1418

1519
"github.com/api7/api7-ingress-controller/internal/controller/config"
1620
)
1721

22+
const (
23+
FinalizerGatewayClassProtection = "apisix.apache.org/gc-protection"
24+
)
25+
1826
// +kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=gatewayclasses,verbs=get;list;watch;update
1927
// +kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=gatewayclasses/status,verbs=get;update
2028

@@ -23,11 +31,13 @@ type GatewayClassReconciler struct { //nolint:revive
2331
client.Client
2432
Scheme *runtime.Scheme
2533

34+
record.EventRecorder
2635
Log logr.Logger
2736
}
2837

2938
// SetupWithManager sets up the controller with the Manager.
3039
func (r *GatewayClassReconciler) SetupWithManager(mgr ctrl.Manager) error {
40+
r.EventRecorder = mgr.GetEventRecorderFor("gatewayclass-controller")
3141
return ctrl.NewControllerManagedBy(mgr).
3242
For(&gatewayv1.GatewayClass{}).
3343
WithEventFilter(predicate.NewPredicateFuncs(r.GatewayClassFilter)).
@@ -41,6 +51,43 @@ func (r *GatewayClassReconciler) Reconcile(ctx context.Context, req ctrl.Request
4151
return ctrl.Result{}, client.IgnoreNotFound(err)
4252
}
4353

54+
if gc.GetDeletionTimestamp().IsZero() {
55+
if !controllerutil.ContainsFinalizer(gc, FinalizerGatewayClassProtection) {
56+
controllerutil.AddFinalizer(gc, FinalizerGatewayClassProtection)
57+
if err := r.Update(ctx, gc); err != nil {
58+
return ctrl.Result{}, err
59+
}
60+
}
61+
} else {
62+
if controllerutil.ContainsFinalizer(gc, FinalizerGatewayClassProtection) {
63+
var gatewayList gatewayv1.GatewayList
64+
if err := r.List(ctx, &gatewayList); err != nil {
65+
r.Log.Error(err, "failed to list gateways")
66+
return ctrl.Result{}, err
67+
}
68+
var gateways []types.NamespacedName
69+
for _, gateway := range gatewayList.Items {
70+
if string(gateway.Spec.GatewayClassName) == gc.GetName() {
71+
gateways = append(gateways, types.NamespacedName{
72+
Namespace: gateway.GetNamespace(),
73+
Name: gateway.GetName(),
74+
})
75+
}
76+
}
77+
if len(gateways) > 0 {
78+
r.Eventf(gc, "Warning", "DeletionBlocked", "the GatewayClass is still used by Gateways: %v", gateways)
79+
return ctrl.Result{RequeueAfter: 5 * time.Second}, nil
80+
} else {
81+
controllerutil.RemoveFinalizer(gc, FinalizerGatewayClassProtection)
82+
if err := r.Update(ctx, gc); err != nil {
83+
return ctrl.Result{}, err
84+
}
85+
}
86+
}
87+
88+
return ctrl.Result{}, nil
89+
}
90+
4491
condition := meta.Condition{
4592
Type: string(gatewayv1.GatewayClassConditionStatusAccepted),
4693
Status: meta.ConditionTrue,

‎test/e2e/gatewayapi/gateway.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,7 @@ spec:
112112
Expect(gcyaml).To(ContainSubstring("message: the gatewayclass has been accepted by the api7-ingress-controller"), "checking GatewayClass condition message")
113113

114114
By("create Gateway")
115-
err = s.CreateResourceFromString(defautlGateway)
115+
err = s.CreateResourceFromStringWithNamespace(defautlGateway, s.CurrentNamespace())
116116
Expect(err).NotTo(HaveOccurred(), "creating Gateway")
117117
time.Sleep(5 * time.Second)
118118

@@ -123,7 +123,7 @@ spec:
123123
Expect(gwyaml).To(ContainSubstring("message: the gateway has been accepted by the api7-ingress-controller"), "checking Gateway condition message")
124124

125125
By("create Gateway with not accepted GatewayClass")
126-
err = s.CreateResourceFromString(noClassGateway)
126+
err = s.CreateResourceFromStringWithNamespace(noClassGateway, s.CurrentNamespace())
127127
Expect(err).NotTo(HaveOccurred(), "creating Gateway")
128128
time.Sleep(5 * time.Second)
129129

@@ -184,7 +184,7 @@ spec:
184184
time.Sleep(5 * time.Second)
185185

186186
By("create Gateway")
187-
err = s.CreateResourceFromString(defaultGateway)
187+
err = s.CreateResourceFromStringWithNamespace(defaultGateway, s.CurrentNamespace())
188188
Expect(err).NotTo(HaveOccurred(), "creating Gateway")
189189
time.Sleep(10 * time.Second)
190190

@@ -257,7 +257,7 @@ spec:
257257
time.Sleep(5 * time.Second)
258258

259259
By("create Gateway")
260-
err = s.CreateResourceFromString(defaultGateway)
260+
err = s.CreateResourceFromStringWithNamespace(defaultGateway, s.CurrentNamespace())
261261
Expect(err).NotTo(HaveOccurred(), "creating Gateway")
262262
time.Sleep(10 * time.Second)
263263

‎test/e2e/gatewayapi/gatewayclass.go

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,18 @@ metadata:
3131
name: api7-not-accepeted
3232
spec:
3333
controllerName: "apisix.apache.org/not-exist"
34+
`
35+
const defaultGateway = `
36+
apiVersion: gateway.networking.k8s.io/v1
37+
kind: Gateway
38+
metadata:
39+
name: api7ee
40+
spec:
41+
gatewayClassName: api7
42+
listeners:
43+
- name: http1
44+
protocol: HTTP
45+
port: 80
3446
`
3547
It("Create GatewayClass", func() {
3648
By("create default GatewayClass")
@@ -53,5 +65,50 @@ spec:
5365
Expect(gcyaml).To(ContainSubstring(`status: Unknown`), "checking GatewayClass condition status")
5466
Expect(gcyaml).To(ContainSubstring("message: Waiting for controller"), "checking GatewayClass condition message")
5567
})
68+
69+
It("Delete GatewayClass", func() {
70+
By("create default GatewayClass")
71+
err := s.CreateResourceFromStringWithNamespace(defautlGatewayClass, "")
72+
Expect(err).NotTo(HaveOccurred(), "creating GatewayClass")
73+
Eventually(func() string {
74+
spec, err := s.GetResourceYaml("GatewayClass", "api7")
75+
Expect(err).NotTo(HaveOccurred(), "get resource yaml")
76+
return spec
77+
}).WithTimeout(8 * time.Second).ProbeEvery(time.Second).Should(ContainSubstring(`status: "True"`))
78+
79+
By("create a Gateway")
80+
err = s.CreateResourceFromStringWithNamespace(defaultGateway, s.CurrentNamespace())
81+
Expect(err).NotTo(HaveOccurred(), "creating Gateway")
82+
time.Sleep(time.Second)
83+
84+
By("try to delete the GatewayClass")
85+
_, err = s.RunKubectlAndGetOutput("delete", "GatewayClass", "api7", "--wait=false")
86+
Expect(err).NotTo(HaveOccurred())
87+
88+
_, err = s.GetResourceYaml("GatewayClass", "api7")
89+
Expect(err).NotTo(HaveOccurred(), "get resource yaml")
90+
91+
output, err := s.RunKubectlAndGetOutput("describe", "GatewayClass", "api7")
92+
Expect(err).NotTo(HaveOccurred(), "describe GatewayClass api7")
93+
Expect(output).To(And(
94+
ContainSubstring("Warning"),
95+
ContainSubstring("DeletionBlocked"),
96+
ContainSubstring("gatewayclass-controller"),
97+
ContainSubstring("the GatewayClass is still used by Gateways"),
98+
))
99+
100+
By("delete the Gateway")
101+
err = s.DeleteResource("Gateway", "api7ee")
102+
Expect(err).NotTo(HaveOccurred(), "deleting Gateway")
103+
time.Sleep(time.Second)
104+
105+
By("try to delete the GatewayClass again")
106+
err = s.DeleteResource("GatewayClass", "api7")
107+
Expect(err).NotTo(HaveOccurred())
108+
109+
_, err = s.GetResourceYaml("GatewayClass", "api7")
110+
Expect(err).To(HaveOccurred(), "get resource yaml")
111+
Expect(err.Error()).To(ContainSubstring("not found"))
112+
})
56113
})
57114
})

‎test/e2e/gatewayapi/gatewayproxy.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -207,7 +207,7 @@ spec:
207207
time.Sleep(5 * time.Second)
208208

209209
By("Create Gateway with GatewayProxy")
210-
err = s.CreateResourceFromString(fmt.Sprintf(gatewayWithProxy, gatewayClassName))
210+
err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(gatewayWithProxy, gatewayClassName), s.CurrentNamespace())
211211
Expect(err).NotTo(HaveOccurred(), "creating Gateway with GatewayProxy")
212212
time.Sleep(5 * time.Second)
213213

@@ -220,9 +220,9 @@ spec:
220220

221221
AfterEach(func() {
222222
By("Clean up resources")
223-
_ = s.DeleteResourceFromString(fmt.Sprintf(gatewayProxyWithEnabledPlugin, framework.DashboardTLSEndpoint, s.AdminKey()))
224223
_ = s.DeleteResourceFromString(fmt.Sprintf(httpRouteForTest, "api7"))
225224
_ = s.DeleteResourceFromString(fmt.Sprintf(gatewayWithProxy, gatewayClassName))
225+
_ = s.DeleteResourceFromString(fmt.Sprintf(gatewayProxyWithEnabledPlugin, framework.DashboardTLSEndpoint, s.AdminKey()))
226226
})
227227

228228
Context("Test Gateway with enabled GatewayProxy plugin", func() {

‎test/e2e/gatewayapi/httproute.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,7 @@ spec:
125125
Expect(gcyaml).To(ContainSubstring("message: the gatewayclass has been accepted by the api7-ingress-controller"), "checking GatewayClass condition message")
126126

127127
By("create Gateway")
128-
err = s.CreateResourceFromString(fmt.Sprintf(defaultGateway, gatewayClassName))
128+
err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(defaultGateway, gatewayClassName), s.CurrentNamespace())
129129
Expect(err).NotTo(HaveOccurred(), "creating Gateway")
130130
time.Sleep(5 * time.Second)
131131

@@ -158,7 +158,7 @@ spec:
158158
Expect(gcyaml).To(ContainSubstring("message: the gatewayclass has been accepted by the api7-ingress-controller"), "checking GatewayClass condition message")
159159

160160
By("create Gateway")
161-
err = s.CreateResourceFromString(fmt.Sprintf(defaultGatewayHTTPS, gatewayClassName))
161+
err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(defaultGatewayHTTPS, gatewayClassName), s.CurrentNamespace())
162162
Expect(err).NotTo(HaveOccurred(), "creating Gateway")
163163
time.Sleep(5 * time.Second)
164164

‎test/e2e/scaffold/k8s.go

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,10 @@ func (s *Scaffold) DeleteResourceFromStringWithNamespace(yaml, namespace string)
129129
return k8s.KubectlDeleteFromStringE(s.t, s.kubectlOptions, yaml)
130130
}
131131

132+
func (s *Scaffold) CurrentNamespace() string {
133+
return s.kubectlOptions.Namespace
134+
}
135+
132136
func (s *Scaffold) NewAPISIX() (dashboard.Dashboard, error) {
133137
return dashboard.NewClient()
134138
}
@@ -268,7 +272,7 @@ func (s *Scaffold) ApplyDefaultGatewayResource(
268272
)
269273

270274
By("create Gateway")
271-
err = s.CreateResourceFromString(fmt.Sprintf(defaultGateway, gatewayClassName))
275+
err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(defaultGateway, gatewayClassName), s.CurrentNamespace())
272276
Expect(err).NotTo(HaveOccurred(), "creating Gateway")
273277
time.Sleep(5 * time.Second)
274278

‎test/e2e/scaffold/scaffold.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -488,8 +488,8 @@ func (s *Scaffold) afterEach() {
488488
err := k8s.DeleteNamespaceE(s.t, s.kubectlOptions, s.namespace)
489489
Expect(err).NotTo(HaveOccurred(), "deleting namespace "+s.namespace)
490490

491-
for _, f := range s.finalizers {
492-
runWithRecover(f)
491+
for i := len(s.finalizers) - 1; i >= 0; i-- {
492+
runWithRecover(s.finalizers[i])
493493
}
494494

495495
// Wait for a while to prevent the worker node being overwhelming

0 commit comments

Comments
 (0)