diff --git a/docs/upgrade-testing.md b/docs/upgrade-testing.md index 68ba33a8..6997c1cf 100644 --- a/docs/upgrade-testing.md +++ b/docs/upgrade-testing.md @@ -240,6 +240,7 @@ Base tests use YAML manifests from `test/upgrade/testdata/baseCrs/`. Currently t - **SpaceMembers** - **ServiceInstance** - **ServiceCredentialBinding** +- ~~**OrgQuota**~~ - To create org quotas, broad [admin privileges](https://docs.cloudfoundry.org/concepts/roles.html#activeroles) are required which we do not want to grant the technical user just for testing purposes so we intentionally leave it out #### Test Base Resource Dependencies - **SpaceRole:** A space role can only be assigned to a user if the user is also a member of the space's organization.\ @@ -248,7 +249,9 @@ Base tests use YAML manifests from `test/upgrade/testdata/baseCrs/`. Currently t If the combination of offering and plan is not available in your space change it something different.\ 🠊 Run `cf marketplace` and update the values in test/upgrade/testdata/baseCrs/service_instance.yaml - **ServiceCredentialBinding:** The ServiceCredentialBinding directly depends on the ServiceInstance it is referencing \ -🠊 The `base_upgrade_test` includes dedicated pre- and post-upgrade assessment for the ServiceInstance resources and its dependents. These assessments verify the ServiceInstance first, and only then thedependent resources such as ServiceCredentialBinding. This ordering makes dependency failures easier to diagnose and test less flaky when the upstream ServiceInstance is not healthy. +🠊 The `base_upgrade_test` includes dedicated pre- and post-upgrade assessment for the ServiceInstance resources and its dependents. These assessments verify the ServiceInstance first, and only then the dependent resources such as ServiceCredentialBinding. This ordering makes dependency failures easier to diagnose and test less flaky when the upstream ServiceInstance is not healthy. +- **Domain, Route & App:** App depends on Route, and Route depends on Domain \ +🠊 The `base_upgrade_test` includes dedicated pre- and post-upgrade assessments for this chain and verifies resources in dependency order (Domain -> Route -> App), skipping dependent resources when a previous dependency fails. This ordering makes dependency failures easier to diagnose and test less flaky when the upstream resources are not healthy. #### Adding New Base Test Resources @@ -368,22 +371,30 @@ test/ ├── upgrade/ │ ├── testdata/ │ │ ├── baseCrs/ # Base upgrade test resources -│ │ │ ├── import/ -│ │ │ │ └── import_org.yaml # Organization (observe) -│ │ │ ├── space/ -│ │ │ │ └── space.yaml # Space (create) +│ │ │ ├── app/ +│ │ │ │ └── app.yaml │ │ │ ├── domain/ -│ │ │ │ └── domain.yaml -│ │ │ ├── spaceQuota/ -│ │ │ │ └── space_quota.yaml -│ │ │ ├── spaceRole/ -│ │ │ │ └── space_role.yaml +│ │ │ │ └── domain.yaml # Organization (observe) + Space (observe) +│ │ │ ├── import/ +│ │ │ │ └── import.yaml +│ │ │ ├── orgMembers/ +│ │ │ │ └── org_members.yaml +│ │ │ ├── orgRole/ +│ │ │ │ └── org_role.yaml +│ │ │ ├── route/ +│ │ │ │ └── route.yaml │ │ │ ├── serviceCredentialBinding/ │ │ │ │ └── service_credential_binding.yaml │ │ │ ├── serviceInstance/ │ │ │ │ └── service_instance.yaml -│ │ │ └── spaceMembers/ -│ │ │ └── space_members.yaml +│ │ │ ├── space/ +│ │ │ │ └── space.yaml # Space (create) +│ │ │ ├── spaceMembers/ +│ │ │ │ └── space_members.yaml +│ │ │ ├── spaceQuota/ +│ │ │ │ └── space_quota.yaml +│ │ │ └── spaceRole/ +│ │ │ └── space_role.yaml │ │ └── customCrs/ # Custom upgrade test resources │ │ └── externalNames/ # External-name validation test │ │ ├── space.yaml diff --git a/test/upgrade/base_upgrade_test.go b/test/upgrade/base_upgrade_test.go index 8ad28822..12048de4 100644 --- a/test/upgrade/base_upgrade_test.go +++ b/test/upgrade/base_upgrade_test.go @@ -33,9 +33,15 @@ func TestUpgradeProvider(t *testing.T) { serviceInstanceDir := filepath.Join(resourceDirectoryRoot, "serviceInstance") serviceCredentialBindingDir := filepath.Join(resourceDirectoryRoot, "serviceCredentialBinding") - dependentDirs := []string{serviceCredentialBindingDir} + serviceInstanceDependencyChain := []string{serviceInstanceDir, serviceCredentialBindingDir} - requiredDirs := removeFromDirs(resourceDirectories, dependentDirs) + domainDir := filepath.Join(resourceDirectoryRoot, "domain") + routeDir := filepath.Join(resourceDirectoryRoot, "route") + appDir := filepath.Join(resourceDirectoryRoot, "app") + domainDependencyChain := []string{domainDir, routeDir, appDir} + + dependencyDirs := append(serviceInstanceDependencyChain, domainDependencyChain...) + requiredDirs := removeFromDirs(resourceDirectories, dependencyDirs) upgradeTest := NewCustomUpgradeTest("baseline-upgrade-test"). FromVersion(fromTag). @@ -49,9 +55,15 @@ func TestUpgradeProvider(t *testing.T) { }, ). WithCustomPreUpgradeAssessment( - "Check service instance and dependent resources are healthy before upgrade", + "Check service instance and service credential binding resources are healthy before upgrade", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - return verifyServiceInstanceWithDependents(ctx, t, cfg, serviceInstanceDir, dependentDirs, verifyTimeout) + return verifyDependencyChain(ctx, t, cfg, serviceInstanceDependencyChain, verifyTimeout) + }, + ). + WithCustomPreUpgradeAssessment( + "Check domain, route, and app resources are healthy before upgrade", + func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + return verifyDependencyChain(ctx, t, cfg, domainDependencyChain, verifyTimeout) }, ). WithCustomPostUpgradeAssessment( @@ -61,9 +73,15 @@ func TestUpgradeProvider(t *testing.T) { }, ). WithCustomPostUpgradeAssessment( - "Check service instance and dependent resources are healthy after upgrade", + "Check service instance and service credential binding resources are healthy after upgrade", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - return verifyServiceInstanceWithDependents(ctx, t, cfg, serviceInstanceDir, dependentDirs, verifyTimeout) + return verifyDependencyChain(ctx, t, cfg, serviceInstanceDependencyChain, verifyTimeout) + }, + ). + WithCustomPostUpgradeAssessment( + "Check domain, route, and app resources are healthy after upgrade", + func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + return verifyDependencyChain(ctx, t, cfg, domainDependencyChain, verifyTimeout) }, ) testenv.Test(t, upgradeTest.Feature()) @@ -81,15 +99,17 @@ func verifyResources(ctx context.Context, t *testing.T, cfg *envconf.Config, dir return ctx } -// verifyServiceInstanceWithDependents verifies the service instance directory first and -// if successful dependent directories -func verifyServiceInstanceWithDependents(ctx context.Context, t *testing.T, cfg *envconf.Config, serviceInstanceDir string, dependentDirs []string, timeout time.Duration) context.Context { - klog.V(4).Infof("verify service instance") - if err := resources.WaitForResourcesToBeSynced(ctx, cfg, serviceInstanceDir, nil, wait.WithTimeout(timeout)); err != nil { - t.Errorf("verify service instance failed: %v — skipping verification of: %s", err, strings.Join(dependentDirs, ", ")) - return ctx +// verifyDependencyChain verifies the resource directories in the order as they appear in the slice +// and skips the remaining if any resource directory in the chain fails verification +func verifyDependencyChain(ctx context.Context, t *testing.T, cfg *envconf.Config, dirs []string, timeout time.Duration) context.Context { + for i, dir := range dirs { + klog.V(4).Infof("verify resources of directory %s", dir) + if err := resources.WaitForResourcesToBeSynced(ctx, cfg, dir, nil, wait.WithTimeout(timeout)); err != nil { + t.Errorf("verify resources of directory %s failed: %v — skipping verification of remaining dependent directories: %s", dir, err, strings.Join(dirs[i+1:], ", ")) + return ctx + } } - return verifyResources(ctx, t, cfg, dependentDirs, timeout) + return ctx } // removeFromDirs is a helper function to remove directories from the list of directories to verify, diff --git a/test/upgrade/testdata/baseCrs/app/app.yaml b/test/upgrade/testdata/baseCrs/app/app.yaml new file mode 100644 index 00000000..98843637 --- /dev/null +++ b/test/upgrade/testdata/baseCrs/app/app.yaml @@ -0,0 +1,25 @@ +apiVersion: cloudfoundry.crossplane.io/v1alpha1 +kind: App +metadata: + name: upgrade-test-app + annotations: + crossplane.io/paused: "false" +spec: + forProvider: + name: upgrade-test-app + spaceRef: + name: upgrade-test-import-space + policy: + resolve: Always + lifecycle: docker + docker: + image: loud/hello_co:latest + processes: + - type: web + health-check-type: http + health-check-http-endpoint: "/" + routes: + - routeRef: + name: upgrade-test-route + policy: + resolve: Always \ No newline at end of file diff --git a/test/upgrade/testdata/baseCrs/orgMembers/org_member.yaml b/test/upgrade/testdata/baseCrs/orgMembers/org_member.yaml new file mode 100644 index 00000000..1a26eed4 --- /dev/null +++ b/test/upgrade/testdata/baseCrs/orgMembers/org_member.yaml @@ -0,0 +1,17 @@ +apiVersion: cloudfoundry.crossplane.io/v1alpha1 +kind: OrgMembers +metadata: + name: upgrade-test-org-members + annotations: + crossplane.io/paused: "false" +spec: + forProvider: + roleType: Managers + members: + - username: upgrade-test-org-member-user1@example.com + - username: upgrade-test-org-member-user2@example.com + orgRef: + name: upgrade-test-org + policy: + resolve: Always + enforcementPolicy: Lax \ No newline at end of file diff --git a/test/upgrade/testdata/baseCrs/orgRole/org_role.yaml b/test/upgrade/testdata/baseCrs/orgRole/org_role.yaml new file mode 100644 index 00000000..7e19657c --- /dev/null +++ b/test/upgrade/testdata/baseCrs/orgRole/org_role.yaml @@ -0,0 +1,15 @@ +# create a new org role with org ref +apiVersion: cloudfoundry.crossplane.io/v1alpha1 +kind: OrgRole +metadata: + name: upgrade-test-org-role + annotations: + crossplane.io/paused: "false" +spec: + forProvider: + type: Auditor + username: upgrade-test-org-role-user@example.com + orgRef: + name: upgrade-test-org + policy: + resolve: Always \ No newline at end of file diff --git a/test/upgrade/testdata/baseCrs/route/route.yaml b/test/upgrade/testdata/baseCrs/route/route.yaml new file mode 100644 index 00000000..83f6f128 --- /dev/null +++ b/test/upgrade/testdata/baseCrs/route/route.yaml @@ -0,0 +1,17 @@ +apiVersion: cloudfoundry.crossplane.io/v1alpha1 +kind: Route +metadata: + name: upgrade-test-route + annotations: + crossplane.io/paused: "false" +spec: + forProvider: + domainRef: + name: upgrade-test-domain + policy: + resolve: Always + host: upgrade-test-route-host + spaceRef: + name: upgrade-test-import-space + policy: + resolve: Always \ No newline at end of file