RHOAIENG-63118 - Add upgrade tests for Kueue resource management

This commit introduces new tests for validating the upgrade process of Kueue resource flavors and cluster queues. The tests ensure that resource specifications and generation numbers are correctly maintained during upgrades. Key functions include `AddUpgradeResourceBaseline`, `StoreUpgradeBaseline`, and `VerifyUpgradeResourceSpecIntegrity`, which handle the storage and verification of resource specifications in ConfigMaps.

Additionally, the tests set up necessary Kueue resources and validate their state before and after the upgrade process, ensuring the integrity of the resource management system.

Co-authored-by: Cursor <cursoragent@cursor.com>

Author: sukumars321

tests/common/support/kueue_upgrade.go

@@ -0,0 +1,88 @@
+/*
+Copyright 2026.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package support
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/onsi/gomega"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	kueuev1beta2 "sigs.k8s.io/kueue/apis/kueue/v1beta2"
+)
+
+const UpgradeRHOAIVersionKey = "rhoai-version"
+
+func AddUpgradeResourceBaseline(data map[string]string, genKey, specKey string, generation int64, spec interface{}) error {
+	specJSON, err := json.Marshal(spec)
+	if err != nil {
+		return err
+	}
+	data[genKey] = fmt.Sprintf("%d", generation)
+	data[specKey] = string(specJSON)
+	return nil
+}
+
+func StoreUpgradeBaseline(test Test, namespace, configMapName string, data map[string]string) {
+	test.T().Helper()
+
+	data[UpgradeRHOAIVersionKey] = GetRHOAIVersionFromDSCI(test)
+	configMap := &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      configMapName,
+			Namespace: namespace,
+		},
+		Data: data,
+	}
+	_ = test.Client().Core().CoreV1().ConfigMaps(namespace).Delete(test.Ctx(), configMapName, metav1.DeleteOptions{})
+	_, err := test.Client().Core().CoreV1().ConfigMaps(namespace).Create(test.Ctx(), configMap, metav1.CreateOptions{})
+	test.Expect(err).NotTo(gomega.HaveOccurred())
+	test.T().Logf("Stored upgrade baseline in ConfigMap %s/%s", namespace, configMapName)
+}
+
+func VerifyUpgradeResourceSpecIntegrity(test Test, resourceName string, generation int64, spec interface{},
+	configMap *corev1.ConfigMap, genKey, specKey string) {
+	test.T().Helper()
+
+	expectedGen := configMap.Data[genKey]
+	actualGen := fmt.Sprintf("%d", generation)
+	if actualGen != expectedGen {
+		currentSpecJSON, _ := json.Marshal(spec)
+		test.T().Logf("%s generation changed during upgrade (%s to %s)", resourceName, expectedGen, actualGen)
+		test.T().Logf("Pre-upgrade %s spec: %s", resourceName, configMap.Data[specKey])
+		test.T().Logf("Post-upgrade %s spec: %s", resourceName, currentSpecJSON)
+	}
+	test.Expect(actualGen).To(gomega.Equal(expectedGen),
+		"%s spec should be unchanged after upgrade (generation %s, expected %s)", resourceName, actualGen, expectedGen)
+	test.T().Logf("%s generation unchanged after upgrade: %s", resourceName, actualGen)
+}
+
+func ClusterQueueNominalGPUQuota(clusterQueue *kueuev1beta2.ClusterQueue, gpuResourceLabel string) resource.Quantity {
+	for _, resourceGroup := range clusterQueue.Spec.ResourceGroups {
+		for _, flavor := range resourceGroup.Flavors {
+			for _, quota := range flavor.Resources {
+				if string(quota.Name) == gpuResourceLabel {
+					return quota.NominalQuota
+				}
+			}
+		}
+	}
+	return resource.Quantity{}
+}

tests/common/support/kueue_upgrade_test.go

@@ -0,0 +1,106 @@
+/*
+Copyright 2026.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package support
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/onsi/gomega"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	kueuev1beta2 "sigs.k8s.io/kueue/apis/kueue/v1beta2"
+)
+
+func TestAddUpgradeResourceBaseline(t *testing.T) {
+	test := NewTest(t)
+
+	data := map[string]string{}
+	spec := kueuev1beta2.ClusterQueueSpec{
+		ResourceGroups: []kueuev1beta2.ResourceGroup{
+			{
+				CoveredResources: []corev1.ResourceName{corev1.ResourceName(NVIDIA.ResourceLabel)},
+				Flavors: []kueuev1beta2.FlavorQuotas{
+					{
+						Resources: []kueuev1beta2.ResourceQuota{
+							{
+								Name:         corev1.ResourceName(NVIDIA.ResourceLabel),
+								NominalQuota: resource.MustParse("1"),
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	err := AddUpgradeResourceBaseline(data, "generation-key", "spec-key", 3, spec)
+	test.Expect(err).NotTo(gomega.HaveOccurred())
+	test.Expect(data["generation-key"]).To(gomega.Equal("3"))
+
+	var restored kueuev1beta2.ClusterQueueSpec
+	err = json.Unmarshal([]byte(data["spec-key"]), &restored)
+	test.Expect(err).NotTo(gomega.HaveOccurred())
+	test.Expect(restored.ResourceGroups).To(gomega.HaveLen(1))
+	test.Expect(restored.ResourceGroups[0].Flavors[0].Resources[0].NominalQuota).To(gomega.Equal(resource.MustParse("1")))
+}
+
+func TestClusterQueueNominalGPUQuota(t *testing.T) {
+	test := NewTest(t)
+
+	clusterQueue := &kueuev1beta2.ClusterQueue{
+		Spec: kueuev1beta2.ClusterQueueSpec{
+			ResourceGroups: []kueuev1beta2.ResourceGroup{
+				{
+					Flavors: []kueuev1beta2.FlavorQuotas{
+						{
+							Resources: []kueuev1beta2.ResourceQuota{
+								{
+									Name:         corev1.ResourceName(NVIDIA.ResourceLabel),
+									NominalQuota: resource.MustParse("2"),
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	quota := ClusterQueueNominalGPUQuota(clusterQueue, NVIDIA.ResourceLabel)
+	test.Expect(quota).To(gomega.Equal(resource.MustParse("2")))
+	test.Expect(ClusterQueueNominalGPUQuota(clusterQueue, "missing")).To(gomega.Equal(resource.Quantity{}))
+}
+
+func TestVerifyUpgradeResourceSpecIntegrity(t *testing.T) {
+	test := NewTest(t)
+	configMap := &corev1.ConfigMap{
+		Data: map[string]string{
+			"gen-key":                "5",
+			"spec-key":               `{"nodeLabels":{"nvidia.com/gpu.present":"true"}}`,
+			UpgradeRHOAIVersionKey: "3.4.0",
+		},
+	}
+	spec := kueuev1beta2.ResourceFlavorSpec{
+		NodeLabels: map[string]string{
+			"nvidia.com/gpu.present": "true",
+		},
+	}
+
+	VerifyUpgradeResourceSpecIntegrity(test, "ResourceFlavor", 5, spec, configMap, "gen-key", "spec-key")
+}