Skip to content

(WIP)chore: Refactor HTTPRoute and HTTPRoutePolicy tests condition assertion #143

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: release-v2-dev
Choose a base branch
from
Open

(WIP)chore: Refactor HTTPRoute and HTTPRoutePolicy tests condition assertion #143

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
56d5b04
chore: Refactor HTTPRoute and HTTPRoutePolicy tests for better reliab…
dspo May 16, 2025
59b3b5e
Add Apache License header to assertion.go
dspo May 16, 2025
186022d
fix:
dspo May 16, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
188 changes: 188 additions & 0 deletions test/e2e/framework/assertion.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,188 @@
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package framework

import (
"context"
"fmt"
"log"
"slices"
"strings"
"time"

"github.com/gruntwork-io/terratest/modules/testing"
"github.com/pkg/errors"
"github.com/stretchr/testify/require"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/wait"
"sigs.k8s.io/controller-runtime/pkg/client"
gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"
"sigs.k8s.io/gateway-api/conformance/utils/kubernetes"

"github.com/apache/apisix-ingress-controller/api/v1alpha1"
)

func GatewayClassMustHaveCondition(t testing.TestingT, cli client.Client, timeout time.Duration, gcNN types.NamespacedName, condition metav1.Condition) {
err := PollUntilGatewayClassMustHaveStatus(cli, timeout, gcNN, func(gc gatewayv1.GatewayClass) bool {
if err := kubernetes.ConditionsHaveLatestObservedGeneration(&gc, gc.Status.Conditions); err != nil {
log.Printf("GatewayClass %s %v", gcNN, err)
return false
}
if findConditionInList(gc.Status.Conditions, condition) {
return true
}
log.Printf("NOT FOUND condition %v in %v", condition, gc.Status.Conditions)
return false
})
require.NoError(t, err, "waiting for GatewayClass to have condition %+v", condition)
}

func PollUntilGatewayClassMustHaveStatus(cli client.Client, timeout time.Duration, gcNN types.NamespacedName, f func(gc gatewayv1.GatewayClass) bool) error {
if err := gatewayv1.Install(cli.Scheme()); err != nil {
return err
}
return wait.PollUntilContextTimeout(context.Background(), time.Second, timeout, true, func(ctx context.Context) (done bool, err error) {
var gc gatewayv1.GatewayClass
if err := cli.Get(ctx, gcNN, &gc); err != nil {
return false, errors.Wrapf(err, "failed to get GatewayClass %s", gcNN)
}
return f(gc), nil
})
}

func GatewayMustHaveCondition(t testing.TestingT, cli client.Client, timeout time.Duration, gwNN types.NamespacedName, condition metav1.Condition) {
err := PollUntilGatewayHaveStatus(cli, timeout, gwNN, func(gw gatewayv1.Gateway) bool {
if err := kubernetes.ConditionsHaveLatestObservedGeneration(&gw, gw.Status.Conditions); err != nil {
log.Printf("Gateway %s %v", gwNN, err)
return false
}
if findConditionInList(gw.Status.Conditions, condition) {
log.Printf("found condition %v in list [%v]", condition, gw.Status.Conditions)
return true
} else {
log.Printf("not found condition %v in list [%v]", condition, gw.Status.Conditions)
return false
}
})
require.NoError(t, err, "waiting for Gateway to have condition %+v", condition)
}

func PollUntilGatewayHaveStatus(cli client.Client, timeout time.Duration, gwNN types.NamespacedName, f func(gateway gatewayv1.Gateway) bool) error {
if err := gatewayv1.Install(cli.Scheme()); err != nil {
return err
}
return wait.PollUntilContextTimeout(context.Background(), time.Second, timeout, true, func(ctx context.Context) (done bool, err error) {
var gw gatewayv1.Gateway
if err := cli.Get(ctx, gwNN, &gw); err != nil {
return false, errors.Wrapf(err, "failed to get Gateway %s", gwNN)
}
return f(gw), nil
})
}

func HTTPRouteMustHaveCondition(t testing.TestingT, cli client.Client, timeout time.Duration, refNN, hrNN types.NamespacedName, condition metav1.Condition) {
err := PollUntilHTTPRouteHaveStatus(cli, timeout, hrNN, func(hr gatewayv1.HTTPRoute) bool {
for _, parent := range hr.Status.Parents {
if err := kubernetes.ConditionsHaveLatestObservedGeneration(&hr, parent.Conditions); err != nil {
log.Printf("HTTPRoute %s (parentRef=%v) %v", hrNN, parentRefToString(parent.ParentRef), err)
return false
}
if (refNN.Name == "" || parent.ParentRef.Name == gatewayv1.ObjectName(refNN.Name)) &&
(refNN.Namespace == "" || (parent.ParentRef.Namespace != nil && string(*parent.ParentRef.Namespace) == refNN.Namespace)) {
if findConditionInList(parent.Conditions, condition) {
log.Printf("found condition %v in list [%v] for %s reference %s", condition, parent.Conditions, hrNN, refNN)
return true
} else {
log.Printf("found condition %v in list [%v] for %s reference %s", condition, parent.Conditions, hrNN, refNN)
}
}
}
return false
})
require.NoError(t, err, "error waiting for HTTPRoute status to have a Condition matching %+v", condition)
}

func PollUntilHTTPRouteHaveStatus(cli client.Client, timeout time.Duration, hrNN types.NamespacedName, f func(route gatewayv1.HTTPRoute) bool) error {
if err := gatewayv1.Install(cli.Scheme()); err != nil {
return err
}
return wait.PollUntilContextTimeout(context.Background(), time.Second, timeout, true, func(ctx context.Context) (done bool, err error) {
var httpRoute gatewayv1.HTTPRoute
if err := cli.Get(ctx, hrNN, &httpRoute); err != nil {
return false, errors.Wrapf(err, "failed to get HTTPRoute %s", hrNN)
}
return f(httpRoute), nil
})
}

func HTTPRoutePolicyMustHaveCondition(t testing.TestingT, client client.Client, timeout time.Duration, refNN, hrpNN types.NamespacedName,
condition metav1.Condition) {
err := PollUntilHTTPRoutePolicyHaveStatus(client, timeout, hrpNN, func(httpRoutePolicy v1alpha1.HTTPRoutePolicy, status v1alpha1.PolicyStatus) bool {
for _, ancestor := range status.Ancestors {
if err := kubernetes.ConditionsHaveLatestObservedGeneration(&httpRoutePolicy, ancestor.Conditions); err != nil {
log.Printf("HTTPRoutePolicy %s (parentRef=%v) %v", hrpNN, parentRefToString(ancestor.AncestorRef), err)
return false
}

if ancestor.AncestorRef.Name == gatewayv1.ObjectName(refNN.Name) &&
(refNN.Namespace == "" || (ancestor.AncestorRef.Namespace != nil && string(*ancestor.AncestorRef.Namespace) == refNN.Namespace)) {
if findConditionInList(ancestor.Conditions, condition) {
log.Printf("found condition %v in list [%v] for %s reference %s", condition, ancestor.Conditions, hrpNN, refNN)
return true
} else {
log.Printf("not found condition %v in list [%v] for %s reference %s", condition, ancestor.Conditions, hrpNN, refNN)
}
}
}
return false
})

require.NoError(t, err, "error waiting for HTTPRoutePolicy status to have a Condition matching %+v", condition)
}

func PollUntilHTTPRoutePolicyHaveStatus(client client.Client, timeout time.Duration, hrpNN types.NamespacedName,
f func(httpRoutePolicy v1alpha1.HTTPRoutePolicy, status v1alpha1.PolicyStatus) bool) error {
_ = v1alpha1.AddToScheme(client.Scheme())
return wait.PollUntilContextTimeout(context.Background(), time.Second, timeout, true, func(ctx context.Context) (done bool, err error) {
var httpRoutePolicy v1alpha1.HTTPRoutePolicy
if err = client.Get(ctx, hrpNN, &httpRoutePolicy); err != nil {
return false, errors.Wrapf(err, "error fetching HTTPRoutePolicy %s", hrpNN)
}
return f(httpRoutePolicy, httpRoutePolicy.Status), nil
})
}

func parentRefToString(p gatewayv1.ParentReference) string {
if p.Namespace != nil && *p.Namespace != "" {
return fmt.Sprintf("%v/%v", p.Namespace, p.Name)
}
return string(p.Name)
}

func findConditionInList(conditions []metav1.Condition, expected metav1.Condition) bool {
return slices.ContainsFunc(conditions, func(item metav1.Condition) bool {
// an empty Status string means "Match any status".
// an empty Reason string means "Match any reason".
if expected.Type == item.Type &&
(expected.Status == "" || expected.Status == item.Status) &&
(expected.Reason == "" || expected.Reason == item.Reason) &&
expected.Message != "" && !strings.Contains(item.Message, expected.Message) {
log.Printf("condition message not match, item.Message: %s, expected.Message: %s", item.Message, expected.Message)
}
return expected.Type == item.Type &&
(expected.Status == "" || expected.Status == item.Status) &&
(expected.Reason == "" || expected.Reason == item.Reason) &&
(expected.Message == "" || strings.Contains(item.Message, expected.Message))
})
}
64 changes: 0 additions & 64 deletions test/e2e/framework/utils.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,32 +20,20 @@ import (
"encoding/json"
"fmt"
"io"
"log"
"net/http"
"net/url"
"slices"
"strings"
"sync"
"time"

"github.com/gavv/httpexpect/v2"
"github.com/gruntwork-io/terratest/modules/testing"
"github.com/onsi/gomega"
"github.com/pkg/errors"
"github.com/stretchr/testify/require"
"golang.org/x/net/html"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/tools/remotecommand"
"k8s.io/utils/ptr"
"sigs.k8s.io/controller-runtime/pkg/client"
gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"
"sigs.k8s.io/gateway-api/conformance/utils/kubernetes"

"github.com/apache/apisix-ingress-controller/api/v1alpha1"
)

func (f *Framework) NewExpectResponse(httpBody any) *httpexpect.Response {
Expand Down Expand Up @@ -381,43 +369,6 @@ func CreateTestZipFile(sourceCode, metadata string) ([]byte, error) {
return zipBuffer.Bytes(), nil
}

func HTTPRoutePolicyMustHaveCondition(t testing.TestingT, client client.Client, timeout time.Duration, refNN, hrpNN types.NamespacedName,
condition metav1.Condition) {
err := EventuallyHTTPRoutePolicyHaveStatus(client, timeout, hrpNN, func(httpRoutePolicy v1alpha1.HTTPRoutePolicy, status v1alpha1.PolicyStatus) bool {
for _, ancestor := range status.Ancestors {
if err := kubernetes.ConditionsHaveLatestObservedGeneration(&httpRoutePolicy, ancestor.Conditions); err != nil {
log.Printf("HTTPRoutePolicy %s (parentRef=%v) %v", hrpNN, parentRefToString(ancestor.AncestorRef), err)
return false
}

if ancestor.AncestorRef.Name == gatewayv1.ObjectName(refNN.Name) &&
(ancestor.AncestorRef.Namespace == nil || refNN.Namespace == "" || string(*ancestor.AncestorRef.Namespace) == refNN.Namespace) {
if findConditionInList(ancestor.Conditions, condition) {
log.Printf("found condition %v in list [%v] for %s reference %s", condition, ancestor.Conditions, hrpNN, refNN)
return true
} else {
log.Printf("not found condition %v in list [%v] for %s reference %s", condition, ancestor.Conditions, hrpNN, refNN)
}
}
}
return false
})

require.NoError(t, err, "error waiting for HTTPRoutePolicy status to have a Condition matching expectations")
}

func EventuallyHTTPRoutePolicyHaveStatus(client client.Client, timeout time.Duration, hrpNN types.NamespacedName,
f func(httpRoutePolicy v1alpha1.HTTPRoutePolicy, status v1alpha1.PolicyStatus) bool) error {
_ = v1alpha1.AddToScheme(client.Scheme())
return wait.PollUntilContextTimeout(context.Background(), time.Second, timeout, true, func(ctx context.Context) (done bool, err error) {
var httpRoutePolicy v1alpha1.HTTPRoutePolicy
if err = client.Get(ctx, hrpNN, &httpRoutePolicy); err != nil {
return false, errors.Errorf("error fetching HTTPRoutePolicy %v: %v", hrpNN, err)
}
return f(httpRoutePolicy, httpRoutePolicy.Status), nil
})
}

func addFileToZip(zipWriter *zip.Writer, fileName, fileContent string) error {
file, err := zipWriter.Create(fileName)
if err != nil {
Expand All @@ -427,18 +378,3 @@ func addFileToZip(zipWriter *zip.Writer, fileName, fileContent string) error {
_, err = file.Write([]byte(fileContent))
return err
}

func parentRefToString(p gatewayv1.ParentReference) string {
if p.Namespace != nil && *p.Namespace != "" {
return fmt.Sprintf("%v/%v", p.Namespace, p.Name)
}
return string(p.Name)
}

func findConditionInList(conditions []metav1.Condition, expected metav1.Condition) bool {
return slices.ContainsFunc(conditions, func(item metav1.Condition) bool {
// an empty Status string means "Match any status".
// an empty Reason string means "Match any reason".
return expected.Type == item.Type && (expected.Status == "" || expected.Status == item.Status) && (expected.Reason == "" || expected.Reason == item.Reason)
})
}
Loading
Loading