From d4044710a50fdfe566a989cd7e15ed4ea9d7d3ff Mon Sep 17 00:00:00 2001 From: matttrach Date: Fri, 11 Apr 2025 18:05:51 -0500 Subject: [PATCH 1/2] fix: filter prime releases from testing Signed-off-by: matttrach --- test/scripts/runningPods.sh | 55 ++++++++++++++++++ test/tests/one/one_test.go | 1 + test/tests/util.go | 107 +++++++++++++++++++++--------------- 3 files changed, 119 insertions(+), 44 deletions(-) create mode 100755 test/scripts/runningPods.sh diff --git a/test/scripts/runningPods.sh b/test/scripts/runningPods.sh new file mode 100755 index 0000000..37c4895 --- /dev/null +++ b/test/scripts/runningPods.sh @@ -0,0 +1,55 @@ +#!/bin/bash +set -x + +JSONPATH="'{range .items[*]} + {.metadata.name}{\"\\t\"} \ + {.metadata.namespace}{\"\\t\"} \ + {.status.phase}{\"\\n\"} \ +{end}'" + +notReady() { + PODS=$(kubectl get pods -A -o jsonpath="$JSONPATH") + # shellcheck disable=SC2060,SC2140 + NOT_READY=$(echo "$PODS" | grep -v "Running" | grep -v "Succeeded" | tr -d ["\t","\n"," ","'"] || true) + if [ -n "$NOT_READY" ]; then + # Some pods aren't running + return 0 + else + # All pods are running + return 1 + fi +} + +TIMEOUT=10 # 10 minutes +TIMEOUT_MINUTES=$((TIMEOUT * 60)) +INTERVAL=30 # 30 seconds +MAX=$((TIMEOUT_MINUTES / INTERVAL)) +INDEX=0 + +while notReady; do + if [[ $INDEX -lt $MAX ]]; then + echo "Waiting for pods to be ready..." + INDEX=$((INDEX + 1)) + sleep $INTERVAL; + else + echo "Timeout reached. Pods are not ready..." + echo "nodes..." + kubectl get nodes || true + echo "all..." + kubectl get all -A || true + echo "pods..." + kubectl get pods -A || true + exit 1 + fi +done + +echo "Pods are ready..." + +echo "nodes..." +kubectl get nodes || true +echo "all..." +kubectl get all -A || true +echo "pods..." +kubectl get pods -A || true + +exit 0 diff --git a/test/tests/one/one_test.go b/test/tests/one/one_test.go index 3e899f8..c97b78f 100644 --- a/test/tests/one/one_test.go +++ b/test/tests/one/one_test.go @@ -103,6 +103,7 @@ func TestOneBasic(t *testing.T) { t.Fatalf("Error creating cluster: %s", err) } util.CheckReady(t, testDir + "/kubeconfig") + util.CheckRunning(t, testDir + "/kubeconfig") if t.Failed() { t.Log("Test failed...") } else { diff --git a/test/tests/util.go b/test/tests/util.go index 86b95fb..4c541ff 100644 --- a/test/tests/util.go +++ b/test/tests/util.go @@ -22,11 +22,13 @@ import ( ) func GetRancherReleases() (string, string, string, error) { - releases, err := getRancherReleases() + releases, err := getReleases("rancher", "rancher") if err != nil { return "", "", "", err } - versions := filterPrerelease(releases) + filterPrerelease(&releases) + filterPrimeOnly(&releases) + versions := getVersionsFromReleases(&releases) if len(versions) == 0 { return "", "", "", errors.New("no eligible versions found") } @@ -46,35 +48,13 @@ func GetRancherReleases() (string, string, string, error) { return latest, stable, lts, nil } -func getRancherReleases() ([]*github.RepositoryRelease, error) { - githubToken := os.Getenv("GITHUB_TOKEN") - if githubToken == "" { - fmt.Println("GITHUB_TOKEN environment variable not set") - return nil, errors.New("GITHUB_TOKEN environment variable not set") - } - - // Create a new OAuth2 token using the GitHub token - tokenSource := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: githubToken}) - tokenClient := oauth2.NewClient(context.Background(), tokenSource) - - // Create a new GitHub client using the authenticated HTTP client - client := github.NewClient(tokenClient) - - var releases []*github.RepositoryRelease - releases, _, err := client.Repositories.ListReleases(context.Background(), "rancher", "rancher", &github.ListOptions{}) - if err != nil { - return nil, err - } - - return releases, nil -} - func GetRke2Releases() (string, string, string, error) { - releases, err := getRke2Releases() + releases, err := getReleases("rancher", "rke2") if err != nil { return "", "", "", err } - versions := filterPrerelease(releases) + filterPrerelease(&releases) + versions := getVersionsFromReleases(&releases) if len(versions) == 0 { return "", "", "", errors.New("no eligible versions found") } @@ -94,7 +74,7 @@ func GetRke2Releases() (string, string, string, error) { return latest, stable, lts, nil } -func getRke2Releases() ([]*github.RepositoryRelease, error) { +func getReleases(org string, repo string) ([]*github.RepositoryRelease, error) { githubToken := os.Getenv("GITHUB_TOKEN") if githubToken == "" { fmt.Println("GITHUB_TOKEN environment variable not set") @@ -109,30 +89,46 @@ func getRke2Releases() ([]*github.RepositoryRelease, error) { client := github.NewClient(tokenClient) var releases []*github.RepositoryRelease - releases, _, err := client.Repositories.ListReleases(context.Background(), "rancher", "rke2", &github.ListOptions{}) + releases, _, err := client.Repositories.ListReleases(context.Background(), org, repo, &github.ListOptions{}) if err != nil { return nil, err } + return releases, nil } -func filterReleaseCandidate(v *[]string) { - var fv []string - versions := *v - for i := 1; i < len(versions); i++ { - if strings.Contains(versions[i], "-") != true { - fv = append(fv, versions[i]) +func filterPrimeOnly(r *[]*github.RepositoryRelease) { + var fr []*github.RepositoryRelease + releases := *r + for i := 0; i < len(releases); i++ { + if len(releases[i].Assets) > 2 { // source zip and tar are always there + // prime only releases won't have artifacts + // so we only add releases with more than 2 artifacts + fr = append(fr, releases[i]) } } - *v = fv + *r = fr +} + +// this effectively removes release candidates as well as pending releases +func filterPrerelease(r *[]*github.RepositoryRelease) { + var fr []*github.RepositoryRelease + releases := *r + for i := 0; i < len(releases); i++ { + if !releases[i].GetPrerelease() { + fr = append(fr, releases[i]) + } + } + *r = fr } -func filterPrerelease(r []*github.RepositoryRelease) []string { +func getVersionsFromReleases(r *[]*github.RepositoryRelease) []string { var versions []string - for _, release := range r { - version := release.GetTagName() - if !release.GetPrerelease() { - versions = append(versions, version) + releases := *r + for i := 0; i < len(releases); i++ { + versions = append(versions, *releases[i].TagName) + } + return versions // [ // "v1.28.14+rke2r1", // "v1.30.1+rke2r3", @@ -145,9 +141,6 @@ func filterPrerelease(r []*github.RepositoryRelease) []string { // "v1.29.5+rke2r1", // "v1.28.17+rke2r1", // ] - } - } - return versions } func sortVersions(v *[]string) { @@ -467,3 +460,29 @@ func CheckReady(t *testing.T, kubeconfigPath string) { } t.Logf("Ready script output: %s", out) } + +func CheckRunning(t *testing.T, kubeconfigPath string) { + repoRoot, err := filepath.Abs(g.GetRepoRoot(t)) + if err != nil { + t.Logf("Error getting git root directory: %v", err) + t.Fail() + } + script, err := os.ReadFile(repoRoot + "/test/scripts/runningPods.sh") + if err != nil { + t.Logf("Error reading script: %v", err) + t.Fail() + } + readyScript := shell.Command{ + Command: "bash", + Args: []string{"-c", string(script)}, + Env: map[string]string{ + "KUBECONFIG": kubeconfigPath, + }, + } + out, err := shell.RunCommandAndGetOutputE(t, readyScript) + if err != nil { + t.Logf("Error running script: %s", err) + t.Fail() + } + t.Logf("Ready script output: %s", out) +} From 71eb76954a49ba1f01bdef7e4f74579565eaebfb Mon Sep 17 00:00:00 2001 From: matttrach Date: Mon, 14 Apr 2025 21:18:21 -0500 Subject: [PATCH 2/2] fix: validate multiple times Signed-off-by: matttrach --- examples/one/main.tf | 10 ++-- examples/one/outputs.tf | 4 ++ examples/prod/main.tf | 28 +++++++++-- examples/prod/outputs.tf | 5 +- modules/rancher_bootstrap/rancher/main.tf | 6 +-- .../rancher_externalTLS/main.tf | 4 +- test/scripts/readyNodes.sh | 6 +-- test/scripts/runningPods.sh | 48 +++++++++++-------- 8 files changed, 73 insertions(+), 38 deletions(-) diff --git a/examples/one/main.tf b/examples/one/main.tf index c9f539a..6736f28 100644 --- a/examples/one/main.tf +++ b/examples/one/main.tf @@ -23,6 +23,9 @@ provider "rancher2" { } resource "rancher2_bootstrap" "authenticate" { + depends_on = [ + module.rancher, + ] provider = rancher2.authenticate initial_password = module.rancher.admin_password password = module.rancher.admin_password @@ -92,12 +95,11 @@ module "rancher" { rancher_version = local.rancher_version } -# test catalog entry -resource "rancher2_catalog" "foo" { +data "rancher2_cluster" "local" { depends_on = [ module.rancher, + rancher2_bootstrap.authenticate, ] provider = rancher2.default - name = "test" - url = "http://foo.com:8080" + name = "local" } diff --git a/examples/one/outputs.tf b/examples/one/outputs.tf index d9eca66..8e218ec 100644 --- a/examples/one/outputs.tf +++ b/examples/one/outputs.tf @@ -16,3 +16,7 @@ output "admin_password" { value = module.rancher.admin_password sensitive = true } +output "cluster_data" { + value = jsonencode(data.rancher2_cluster.local) + sensitive = true +} diff --git a/examples/prod/main.tf b/examples/prod/main.tf index a11efa4..ac52e7d 100644 --- a/examples/prod/main.tf +++ b/examples/prod/main.tf @@ -16,11 +16,31 @@ provider "kubernetes" {} # make sure you set the env variable KUBE_CONFIG_PATH t provider "helm" {} # make sure you set the env variable KUBE_CONFIG_PATH to local_file_path (file_path variable) provider "rancher2" { + alias = "default" api_url = "https://${local.domain}.${local.zone}" token_key = module.this.admin_token timeout = "300s" } +provider "rancher2" { + alias = "authenticate" + bootstrap = true + api_url = "https://${local.domain}.${local.zone}" + timeout = "300s" +} + +resource "rancher2_bootstrap" "authenticate" { + depends_on = [ + module.this, + ] + provider = rancher2.authenticate + initial_password = module.this.admin_password + password = module.this.admin_password + token_update = true + token_ttl = 7200 # 2 hours +} + + locals { identifier = var.identifier example = "prod" @@ -149,11 +169,11 @@ resource "terraform_data" "get_cert_info" { } } -# test catalog entry -resource "rancher2_catalog" "foo" { +data "rancher2_cluster" "local" { depends_on = [ module.this, + rancher2_bootstrap.authenticate, ] - name = "test" - url = "http://foo.com:8080" + provider = rancher2.default + name = "local" } diff --git a/examples/prod/outputs.tf b/examples/prod/outputs.tf index f0cd0f3..ff1955e 100644 --- a/examples/prod/outputs.tf +++ b/examples/prod/outputs.tf @@ -16,4 +16,7 @@ output "admin_password" { value = module.this.admin_password sensitive = true } - +output "cluster_data" { + value = jsonencode(data.rancher2_cluster.local) + sensitive = true +} diff --git a/modules/rancher_bootstrap/rancher/main.tf b/modules/rancher_bootstrap/rancher/main.tf index 5745b6e..60d25b0 100644 --- a/modules/rancher_bootstrap/rancher/main.tf +++ b/modules/rancher_bootstrap/rancher/main.tf @@ -150,10 +150,10 @@ resource "helm_release" "rancher" { chart = "${path.root}/rancher-${local.rancher_version}.tgz" # "${local.rancher_helm_repository}/${local.rancher_channel}/rancher-${local.rancher_version}.tgz" namespace = "cattle-system" create_namespace = false - wait = false - wait_for_jobs = false + wait = true + wait_for_jobs = true force_update = true - timeout = 1800 # 30m + timeout = 3600 # 60m set { name = "hostname" diff --git a/modules/rancher_bootstrap/rancher_externalTLS/main.tf b/modules/rancher_bootstrap/rancher_externalTLS/main.tf index be7ce67..6154a1f 100644 --- a/modules/rancher_bootstrap/rancher_externalTLS/main.tf +++ b/modules/rancher_bootstrap/rancher_externalTLS/main.tf @@ -67,8 +67,8 @@ resource "helm_release" "rancher" { chart = "${path.root}/rancher-${local.rancher_version}.tgz" #"${local.rancher_helm_repository}/${local.rancher_channel}/rancher-${local.rancher_version}.tgz" namespace = "cattle-system" create_namespace = false - wait = false - wait_for_jobs = false + wait = true + wait_for_jobs = true force_update = true timeout = 1800 # 30m diff --git a/test/scripts/readyNodes.sh b/test/scripts/readyNodes.sh index 6c6d3ad..983b6c4 100755 --- a/test/scripts/readyNodes.sh +++ b/test/scripts/readyNodes.sh @@ -32,12 +32,12 @@ TIMEOUT=5 # 5 minutes TIMEOUT_MINUTES=$((TIMEOUT * 60)) INTERVAL=10 # 10 seconds MAX=$((TIMEOUT_MINUTES / INTERVAL)) -INDEX=0 +ATTEMPTS=0 while notReady; do - if [[ $INDEX -lt $MAX ]]; then + if [[ $ATTEMPTS -lt $MAX ]]; then echo "Waiting for nodes to be ready..." - INDEX=$((INDEX + 1)) + ATTEMPTS=$((ATTEMPTS + 1)) sleep $INTERVAL; else echo "Timeout reached. Nodes are not ready..." diff --git a/test/scripts/runningPods.sh b/test/scripts/runningPods.sh index 37c4895..3d745c5 100755 --- a/test/scripts/runningPods.sh +++ b/test/scripts/runningPods.sh @@ -20,35 +20,41 @@ notReady() { fi } -TIMEOUT=10 # 10 minutes -TIMEOUT_MINUTES=$((TIMEOUT * 60)) -INTERVAL=30 # 30 seconds -MAX=$((TIMEOUT_MINUTES / INTERVAL)) -INDEX=0 - -while notReady; do - if [[ $INDEX -lt $MAX ]]; then - echo "Waiting for pods to be ready..." - INDEX=$((INDEX + 1)) - sleep $INTERVAL; - else - echo "Timeout reached. Pods are not ready..." - echo "nodes..." - kubectl get nodes || true - echo "all..." - kubectl get all -A || true - echo "pods..." - kubectl get pods -A || true - exit 1 - fi +readyWait() { + TIMEOUT=10 # 10 minutes + TIMEOUT_MINUTES=$((TIMEOUT * 60)) + INTERVAL=30 # 30 seconds + MAX=$((TIMEOUT_MINUTES / INTERVAL)) + ATTEMPTS=0 + + while notReady; do + if [ "$ATTEMPTS" -lt "$MAX" ]; then + ATTEMPTS=$((ATTEMPTS + 1)) + sleep "$INTERVAL"; + else + return 1 + fi + done + return 0 +} + +SUCCESSES=0 +SUCCESSES_NEEDED=3 # require three successes to make sure everything is settled + +while readyWait && [ "$SUCCESSES" -lt "$SUCCESSES_NEEDED" ]; do + SUCCESSES=$((SUCCESSES + 1)) + echo "succeeeded $SUCCESSES times..." + sleep 30 done echo "Pods are ready..." echo "nodes..." kubectl get nodes || true + echo "all..." kubectl get all -A || true + echo "pods..." kubectl get pods -A || true