feat[168]: Add batch 2 upgrade test resources

docs/upgrade-testing.md

@@ -60,9 +60,15 @@ You need valid CloudFoundry credentials with appropriate permissions:
 
 ## Quick Start
 
-### ⚠️ IMPORTANT: Configure Your CF Organization First
+### ⚠️ IMPORTANT: Necessary configuration steps before running any tests
 
-Before running any tests, you **must** update the organization name to one you have access to:
+Some configuration steps are necessary before you can successfully run any upgrade tests:
+- Configure your CF organization
+- Configure your CF space
+
+#### CF Organization
+
+Before running any tests, you **must** update the organization name to one you have access:
 
 1. **List your available CF organizations:**
 ```bash
@@ -85,6 +91,42 @@ spec:
     name: your-org-name-here  # ← Change this to your CF org name
 ```
 
+#### CF Space
+
+Before running the base tests, you **must** update the space name to one in your organization.
+That can either be an existing one you have atleast the SpaceDeveloper role in or you create a new one as describe below:
+
+1. **Optionally create and configure a new CF Space**
+
+Create a space and give your user the SpaceDeveloper role
+```bash
+cf create-space <SPACE_NAME> -o <ORG_NAME>  # Create a space in your org
+cf set-space-role <USERNAME> <ORG_NAME> <SPACE_NAME> SpaceDeveloper # Assign your user the SpaceDeveloper role
+```
+
+2. **List your available CF spaces:**
+```bash
+cf spaces # List spaces
+```
+3. **Update the spaces name** in test manifests: 
+- For base tests: `test/upgrade/testdata/baseCrs/import.yaml`:
+- For custom tests: `test/upgrade/testdata/customCRs/*/import.yaml` (if applicable)
+```yaml
+apiVersion: cloudfoundry.crossplane.io/v1alpha1
+kind: Space
+metadata:
+  name: upgrade-test-import-space
+spec:
+  managementPolicies:
+    - Observe
+  forProvider:
+    name: upgrade-test-space-donotdelete  # ← Change this to you CF space name
+    orgRef:
+      name: upgrade-test-org 
+  providerConfigRef:
+    name: default
+```
+
 ### 1. Set Environment Variables
 
 #### Option A: Use the provided template 
@@ -190,14 +232,23 @@ make test-upgrade-custom
 Base tests use YAML manifests from `test/upgrade/testdata/baseCrs/`. Currently tested resources:
 
 - **Organization** (import) - Uses `managementPolicies: [Observe]` to import existing org
+- **Space** (import) - Uses `managementPolicies: [Observe]` to import existing space
 - **Space** - Lightweight resource for testing basic upgrade flow
 - **Domain**
 - **SpaceQuota**
 - **SpaceRole**
+- **SpaceMembers**
+- **ServiceInstance**
+- **ServiceCredentialBinding**
 
 #### Test Base Resource Dependencies
 - **SpaceRole:** A space role can only be assigned to a user if the user is also a member of the space's organization.\
 🠊 Assign a user to the space's organization by either creating a SpaceMembers/SpaceRole resource or by using the BTP Cockpit
+- **ServiceInstance:** A managed service instance requires a ServicePlan specifying an offering and a plan.
+If the combination of offering and plan is not available in your space change it something different.\
+🠊 Run `cf marketplace` and update the values in test/upgrade/testdata/baseCrs/service_instance.yaml
+- **ServiceCredentialBinding:** The ServiceCredentialBinding directly depends on the ServiceInstance it is referencing \
+🠊 The `base_upgrade_test` includes dedicated pre- and post-upgrade assessment for the ServiceInstance resources and its dependents. These assessments verify the ServiceInstance first, and only then thedependent resources such as ServiceCredentialBinding. This ordering makes dependency failures easier to diagnose and test less flaky when the upstream ServiceInstance is not healthy.
 
 #### Adding New Base Test Resources
 
@@ -317,15 +368,26 @@ test/
 ├── upgrade/
 │   ├── testdata/
 │   │   ├── baseCrs/                      # Base upgrade test resources
-│   │   │   ├── import.yaml               # Organization (observe)
-│   │   │   ├── space.yaml                # Space (create)
-|   │   │   ├── domain.yaml
-|   │   │   ├── space_quota.yaml
-|   │   │   └── space_role.yaml
-│   │   └── customCRs/                    # Custom upgrade test resources
+│   │   │   ├── import/
+│   │   │   │   └── import_org.yaml       # Organization (observe)
+│   │   │   ├── space/
+│   │   │   │   └── space.yaml            # Space (create)
+│   │   │   ├── domain/
+│   │   │   │   └── domain.yaml
+│   │   │   ├── spaceQuota/
+│   │   │   │   └── space_quota.yaml
+│   │   │   ├── spaceRole/
+│   │   │   │   └── space_role.yaml
+│   │   │   ├── serviceCredentialBinding/
+│   │   │   │   └── service_credential_binding.yaml
+│   │   │   ├── serviceInstance/
+│   │   │   │   └── service_instance.yaml
+│   │   │   └── spaceMembers/
+│   │   │       └── space_members.yaml
+│   │   └── customCrs/                    # Custom upgrade test resources
 │   │       └── externalNames/            # External-name validation test
 │   │           ├── space.yaml
-|   |           └── import.yaml
+│   │           └── import.yaml
 │   ├── main_test.go                      # Test environment setup
 │   ├── upgrade_test.go                   # Base upgrade test logic
 │   ├── base_upgrade_test.go              # Custom upgrade test framework

test/upgrade/base_upgrade_test.go

@@ -15,20 +15,92 @@
 package upgrade
 
 import (
+	"context"
+	"path/filepath"
+	"slices"
+	"strings"
 	"testing"
+	"time"
+
+	"github.com/crossplane-contrib/xp-testing/pkg/resources"
+	"k8s.io/klog/v2"
+	"sigs.k8s.io/e2e-framework/klient/wait"
+	"sigs.k8s.io/e2e-framework/pkg/envconf"
 )
 
 func TestUpgradeProvider(t *testing.T) {
 	fromTag, toTag := loadTags()
 
+	serviceInstanceDir := filepath.Join(resourceDirectoryRoot, "serviceInstance")
+	serviceCredentialBindingDir := filepath.Join(resourceDirectoryRoot, "serviceCredentialBinding")
+	dependentDirs := []string{serviceCredentialBindingDir}
+
+	requiredDirs := removeFromDirs(resourceDirectories, dependentDirs)
+
 	upgradeTest := NewCustomUpgradeTest("baseline-upgrade-test").
 		FromVersion(fromTag).
 		ToVersion(toTag).
-		WithResourceDirectories(resourceDirectories)
-
+		WithResourceDirectories(resourceDirectories).
+		SkipDefaultResourceVerification().
+		WithCustomPreUpgradeAssessment(
+			"Check all required resources are healthy before upgrade",
+			func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
+				return verifyResources(ctx, t, cfg, requiredDirs, verifyTimeout)
+			},
+		).
+		WithCustomPreUpgradeAssessment(
+			"Check service instance and dependent resources are healthy before upgrade",
+			func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
+				return verifyServiceInstanceWithDependents(ctx, t, cfg, serviceInstanceDir, dependentDirs, verifyTimeout)
+			},
+		).
+		WithCustomPostUpgradeAssessment(
+			"Check all required resources are healthy after upgrade",
+			func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
+				return verifyResources(ctx, t, cfg, requiredDirs, verifyTimeout)
+			},
+		).
+		WithCustomPostUpgradeAssessment(
+			"Check service instance and dependent resources are healthy after upgrade",
+			func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
+				return verifyServiceInstanceWithDependents(ctx, t, cfg, serviceInstanceDir, dependentDirs, verifyTimeout)
+			},
+		)
 	testenv.Test(t, upgradeTest.Feature())
 }
 
+// verifyResources waits for resources in dirs to be ready
+func verifyResources(ctx context.Context, t *testing.T, cfg *envconf.Config, dirs []string, timeout time.Duration) context.Context {
+	for _, dir := range dirs {
+		klog.V(4).Infof("verify resources of directory %s", dir)
+		if err := resources.WaitForResourcesToBeSynced(ctx, cfg, dir, nil, wait.WithTimeout(timeout)); err != nil {
+			t.Errorf("verify resources of directory %s failed: %v", dir, err)
+		}
+	}
+
+	return ctx
+}
+
+// verifyServiceInstanceWithDependents verifies the service instance directory first and
+// if successful dependent directories
+func verifyServiceInstanceWithDependents(ctx context.Context, t *testing.T, cfg *envconf.Config, serviceInstanceDir string, dependentDirs []string, timeout time.Duration) context.Context {
+	klog.V(4).Infof("verify service instance")
+	if err := resources.WaitForResourcesToBeSynced(ctx, cfg, serviceInstanceDir, nil, wait.WithTimeout(timeout)); err != nil {
+		t.Errorf("verify service instance failed: %v — skipping verification of: %s", err, strings.Join(dependentDirs, ", "))
+		return ctx
+	}
+	return verifyResources(ctx, t, cfg, dependentDirs, timeout)
+}
+
+// removeFromDirs is a helper function to remove directories from the list of directories to verify,
+// allowing dependent resources to be verified separately
+func removeFromDirs(dirs []string, remove []string) []string {
+	result := slices.Clone(dirs)
+	return slices.DeleteFunc(result, func(d string) bool {
+		return slices.Contains(remove, d)
+	})
+}
+
 // loadTags is a helper function to load FROM and TO tags for tests
 // This allows custom tests to reuse the same version configuration
 func loadTags() (string, string) {

test/upgrade/testdata/baseCrs/domain.yaml → test/upgrade/testdata/baseCrs/domain/domain.yaml

@@ -2,6 +2,8 @@ apiVersion: cloudfoundry.crossplane.io/v1alpha1
 kind: Domain
 metadata:
   name: upgrade-test-domain
+  annotations:
+    crossplane.io/paused: "false"
 spec:
   forProvider:
     name: cfupgradetest.eu12.hana.ondemand.com

test/upgrade/testdata/baseCrs/import/import.yaml

@@ -0,0 +1,31 @@
+apiVersion: cloudfoundry.crossplane.io/v1alpha1
+kind: Organization
+metadata:
+  name: upgrade-test-org
+  annotations:
+    crossplane.io/paused: "false"
+spec:
+  managementPolicies:
+    - Observe
+  forProvider:
+    # IMPORTANT: Change this to an org you have access to
+    # Run `cf orgs` to see available organizations
+    name: cf-ci-e2e # ← TODO: Change this to an existing org name
+  providerConfigRef:
+    name: default
+---
+apiVersion: cloudfoundry.crossplane.io/v1alpha1
+kind: Space
+metadata:
+  name: upgrade-test-import-space
+  annotations:
+    crossplane.io/paused: "false"
+spec:
+  managementPolicies:
+    - Observe
+  forProvider:
+    name: upgrade-test-space-donotdelete
+    orgRef:
+      name: upgrade-test-org  # ← References the imported org
+  providerConfigRef:
+    name: default

test/upgrade/testdata/baseCrs/serviceCredentialBinding/service_credential_binding.yaml

@@ -0,0 +1,16 @@
+apiVersion: cloudfoundry.crossplane.io/v1alpha1
+kind: ServiceCredentialBinding
+metadata:
+  name: upgrade-test-service-credential-binding
+  annotations:
+    crossplane.io/paused: "false"
+spec:
+  forProvider:
+    name: upgrade-test-service-credential-binding
+    type: key
+    serviceInstanceRef:
+      name: upgrade-test-service-instance
+      policy:
+        resolve: Always
+  providerConfigRef:
+    name: default

test/upgrade/testdata/baseCrs/serviceInstance/service_instance.yaml

@@ -0,0 +1,19 @@
+apiVersion: cloudfoundry.crossplane.io/v1alpha1
+kind: ServiceInstance
+metadata:
+  name: upgrade-test-service-instance
+  annotations:
+    crossplane.io/paused: "false"
+spec:
+  forProvider:
+    type: managed
+    name: upgrade-test-service-instance
+    spaceRef:
+      name: upgrade-test-import-space
+      policy:
+        resolve: Always
+    servicePlan:
+      offering: destination # Change the offering if not available
+      plan: lite            # Change the plan if not available
+  providerConfigRef:
+    name: default

test/upgrade/testdata/baseCrs/space.yaml → test/upgrade/testdata/baseCrs/space/space.yaml

@@ -2,6 +2,8 @@ apiVersion: cloudfoundry.crossplane.io/v1alpha1
 kind: Space
 metadata:
   name: upgrade-test-space
+  annotations:
+    crossplane.io/paused: "false"
 spec:
   forProvider:
     name: upgrade-test-space

test/upgrade/testdata/baseCrs/spaceMembers/space_members.yaml

@@ -0,0 +1,19 @@
+apiVersion: cloudfoundry.crossplane.io/v1alpha1
+kind: SpaceMembers
+metadata:
+  name: upgrade-test-space-members
+  annotations:
+    crossplane.io/paused: "false"
+spec:
+  forProvider:
+    roleType: Developers
+    members:
+    - username: upgrade-test-space-member-1@example.com
+    - username: upgrade-test-space-member-2@example.com
+    spaceRef:
+      name: upgrade-test-import-space
+      policy:
+        resolve: Always
+    enforcementPolicy: Lax
+  providerConfigRef:
+    name: default

test/upgrade/testdata/baseCrs/space_quota.yaml → test/upgrade/testdata/baseCrs/spaceQuota/space_quota.yaml

@@ -2,6 +2,8 @@ apiVersion: cloudfoundry.crossplane.io/v1alpha1
 kind: SpaceQuota
 metadata:
   name: upgrade-test-space-quota
+  annotations:
+    crossplane.io/paused: "false"
 spec:
   forProvider:
     allowPaidServicePlans: false

test/upgrade/testdata/baseCrs/space_role.yaml → test/upgrade/testdata/baseCrs/spaceRole/space_role.yaml

@@ -2,10 +2,12 @@ apiVersion: cloudfoundry.crossplane.io/v1alpha1
 kind: SpaceRole
 metadata:
   name: upgrade-test-space-role
+  annotations:
+    crossplane.io/paused: "false"
 spec:
   forProvider:
     type: Developer
-    username: upgrade-test-user@example.com
+    username: upgrade-test-space-role@example.com
     spaceRef:
       name: upgrade-test-space
       policy: