fix: filter prime releases from testing (rancher#88)

Signed-off-by: matttrach <matt.trachier@suse.com>

Author: matttrach

examples/one/main.tf

@@ -23,6 +23,9 @@ provider "rancher2" {
 }
 
 resource "rancher2_bootstrap" "authenticate" {
+  depends_on = [
+    module.rancher,
+  ]
   provider         = rancher2.authenticate
   initial_password = module.rancher.admin_password
   password         = module.rancher.admin_password
@@ -92,12 +95,11 @@ module "rancher" {
   rancher_version      = local.rancher_version
 }
 
-# test catalog entry
-resource "rancher2_catalog" "foo" {
+data "rancher2_cluster" "local" {
   depends_on = [
     module.rancher,
+    rancher2_bootstrap.authenticate,
   ]
   provider = rancher2.default
-  name     = "test"
-  url      = "http://foo.com:8080"
+  name     = "local"
 }

examples/one/outputs.tf

@@ -16,3 +16,7 @@ output "admin_password" {
   value     = module.rancher.admin_password
   sensitive = true
 }
+output "cluster_data" {
+  value     = jsonencode(data.rancher2_cluster.local)
+  sensitive = true
+}

examples/prod/main.tf

@@ -16,11 +16,31 @@ provider "kubernetes" {} # make sure you set the env variable KUBE_CONFIG_PATH t
 provider "helm" {}       # make sure you set the env variable KUBE_CONFIG_PATH to local_file_path (file_path variable)
 
 provider "rancher2" {
+  alias     = "default"
   api_url   = "https://${local.domain}.${local.zone}"
   token_key = module.this.admin_token
   timeout   = "300s"
 }
 
+provider "rancher2" {
+  alias     = "authenticate"
+  bootstrap = true
+  api_url   = "https://${local.domain}.${local.zone}"
+  timeout   = "300s"
+}
+
+resource "rancher2_bootstrap" "authenticate" {
+  depends_on = [
+    module.this,
+  ]
+  provider         = rancher2.authenticate
+  initial_password = module.this.admin_password
+  password         = module.this.admin_password
+  token_update     = true
+  token_ttl        = 7200 # 2 hours
+}
+
+
 locals {
   identifier           = var.identifier
   example              = "prod"
@@ -149,11 +169,11 @@ resource "terraform_data" "get_cert_info" {
   }
 }
 
-# test catalog entry
-resource "rancher2_catalog" "foo" {
+data "rancher2_cluster" "local" {
   depends_on = [
     module.this,
+    rancher2_bootstrap.authenticate,
   ]
-  name = "test"
-  url  = "http://foo.com:8080"
+  provider = rancher2.default
+  name     = "local"
 }

examples/prod/outputs.tf

@@ -16,4 +16,7 @@ output "admin_password" {
   value     = module.this.admin_password
   sensitive = true
 }
-
+output "cluster_data" {
+  value     = jsonencode(data.rancher2_cluster.local)
+  sensitive = true
+}

modules/rancher_bootstrap/rancher/main.tf

@@ -150,10 +150,10 @@ resource "helm_release" "rancher" {
   chart            = "${path.root}/rancher-${local.rancher_version}.tgz" # "${local.rancher_helm_repository}/${local.rancher_channel}/rancher-${local.rancher_version}.tgz"
   namespace        = "cattle-system"
   create_namespace = false
-  wait             = false
-  wait_for_jobs    = false
+  wait             = true
+  wait_for_jobs    = true
   force_update     = true
-  timeout          = 1800 # 30m
+  timeout          = 3600 # 60m
 
   set {
     name  = "hostname"

modules/rancher_bootstrap/rancher_externalTLS/main.tf

@@ -67,8 +67,8 @@ resource "helm_release" "rancher" {
   chart            = "${path.root}/rancher-${local.rancher_version}.tgz" #"${local.rancher_helm_repository}/${local.rancher_channel}/rancher-${local.rancher_version}.tgz"
   namespace        = "cattle-system"
   create_namespace = false
-  wait             = false
-  wait_for_jobs    = false
+  wait             = true
+  wait_for_jobs    = true
   force_update     = true
   timeout          = 1800 # 30m
 

test/scripts/readyNodes.sh

@@ -32,12 +32,12 @@ TIMEOUT=5 # 5 minutes
 TIMEOUT_MINUTES=$((TIMEOUT * 60))
 INTERVAL=10 # 10 seconds
 MAX=$((TIMEOUT_MINUTES / INTERVAL))
-INDEX=0
+ATTEMPTS=0
 
 while notReady; do
-  if [[ $INDEX -lt $MAX ]]; then
+  if [[ $ATTEMPTS -lt $MAX ]]; then
     echo "Waiting for nodes to be ready..."
-    INDEX=$((INDEX + 1))
+    ATTEMPTS=$((ATTEMPTS + 1))
     sleep $INTERVAL;
   else
     echo "Timeout reached. Nodes are not ready..."

test/scripts/runningPods.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+set -x
+
+JSONPATH="'{range .items[*]}
+  {.metadata.name}{\"\\t\"} \
+  {.metadata.namespace}{\"\\t\"} \
+  {.status.phase}{\"\\n\"} \
+{end}'"
+
+notReady() {
+  PODS=$(kubectl get pods -A -o jsonpath="$JSONPATH")
+  # shellcheck disable=SC2060,SC2140
+  NOT_READY=$(echo "$PODS" | grep -v "Running" | grep -v "Succeeded"  | tr -d ["\t","\n"," ","'"] || true)
+  if [ -n "$NOT_READY" ]; then
+    # Some pods aren't running
+    return 0
+  else
+    # All pods are running
+    return 1
+  fi
+}
+
+readyWait() {
+  TIMEOUT=10 # 10 minutes
+  TIMEOUT_MINUTES=$((TIMEOUT * 60))
+  INTERVAL=30 # 30 seconds
+  MAX=$((TIMEOUT_MINUTES / INTERVAL))
+  ATTEMPTS=0
+
+  while notReady; do
+    if [ "$ATTEMPTS" -lt "$MAX" ]; then
+      ATTEMPTS=$((ATTEMPTS + 1))
+      sleep "$INTERVAL";
+    else
+      return 1
+    fi
+  done
+  return 0
+}
+
+SUCCESSES=0
+SUCCESSES_NEEDED=3 # require three successes to make sure everything is settled
+
+while readyWait && [ "$SUCCESSES" -lt "$SUCCESSES_NEEDED" ]; do
+  SUCCESSES=$((SUCCESSES + 1))
+  echo "succeeeded $SUCCESSES times..."
+  sleep 30
+done
+
+echo "Pods are ready..."
+
+echo "nodes..."
+kubectl get nodes || true
+
+echo "all..."
+kubectl get all -A || true
+
+echo "pods..."
+kubectl get pods -A || true
+
+exit 0

test/tests/one/one_test.go

@@ -103,6 +103,7 @@ func TestOneBasic(t *testing.T) {
     t.Fatalf("Error creating cluster: %s", err)
   }
   util.CheckReady(t, testDir + "/kubeconfig")
+  util.CheckRunning(t, testDir + "/kubeconfig")
   if t.Failed() {
     t.Log("Test failed...")
   } else {

test/tests/util.go

@@ -22,11 +22,13 @@ import (
 )
 
 func GetRancherReleases() (string, string, string, error) {
-  releases, err := getRancherReleases()
+  releases, err := getReleases("rancher", "rancher")
   if err != nil {
     return "", "", "", err
   }
-  versions := filterPrerelease(releases)
+  filterPrerelease(&releases)
+  filterPrimeOnly(&releases)
+  versions := getVersionsFromReleases(&releases)
   if len(versions) == 0 {
     return "", "", "", errors.New("no eligible versions found")
   }
@@ -46,35 +48,13 @@ func GetRancherReleases() (string, string, string, error) {
   return latest, stable, lts, nil
 }
 
-func getRancherReleases() ([]*github.RepositoryRelease, error) {
-  githubToken := os.Getenv("GITHUB_TOKEN")
-  if githubToken == "" {
-    fmt.Println("GITHUB_TOKEN environment variable not set")
-    return nil, errors.New("GITHUB_TOKEN environment variable not set")
-  }
-
-  // Create a new OAuth2 token using the GitHub token
-  tokenSource := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: githubToken})
-  tokenClient := oauth2.NewClient(context.Background(), tokenSource)
-
-  // Create a new GitHub client using the authenticated HTTP client
-  client := github.NewClient(tokenClient)
-
-  var releases []*github.RepositoryRelease
-  releases, _, err := client.Repositories.ListReleases(context.Background(), "rancher", "rancher", &github.ListOptions{})
-  if err != nil {
-    return nil, err
-  }
-
-  return releases, nil
-}
-
 func GetRke2Releases() (string, string, string, error) {
-  releases, err := getRke2Releases()
+  releases, err := getReleases("rancher", "rke2")
   if err != nil {
     return "", "", "", err
   }
-  versions := filterPrerelease(releases)
+  filterPrerelease(&releases)
+  versions := getVersionsFromReleases(&releases)
   if len(versions) == 0 {
     return "", "", "", errors.New("no eligible versions found")
   }
@@ -94,7 +74,7 @@ func GetRke2Releases() (string, string, string, error) {
   return latest, stable, lts, nil
 }
 
-func getRke2Releases() ([]*github.RepositoryRelease, error) {
+func getReleases(org string, repo string) ([]*github.RepositoryRelease, error) {
   githubToken := os.Getenv("GITHUB_TOKEN")
   if githubToken == "" {
     fmt.Println("GITHUB_TOKEN environment variable not set")
@@ -109,30 +89,46 @@ func getRke2Releases() ([]*github.RepositoryRelease, error) {
   client := github.NewClient(tokenClient)
 
   var releases []*github.RepositoryRelease
-  releases, _, err := client.Repositories.ListReleases(context.Background(), "rancher", "rke2", &github.ListOptions{})
+  releases, _, err := client.Repositories.ListReleases(context.Background(), org, repo, &github.ListOptions{})
   if err != nil {
     return nil, err
   }
+
   return releases, nil
 }
 
-func filterReleaseCandidate(v *[]string) {
-  var fv []string
-  versions := *v
-  for i := 1; i < len(versions); i++ {
-    if strings.Contains(versions[i], "-") != true {
-      fv = append(fv, versions[i])
+func filterPrimeOnly(r *[]*github.RepositoryRelease) {
+  var fr []*github.RepositoryRelease
+  releases := *r
+  for i := 0; i < len(releases); i++ {
+    if len(releases[i].Assets) > 2 { // source zip and tar are always there
+      // prime only releases won't have artifacts
+      // so we only add releases with more than 2 artifacts
+      fr = append(fr, releases[i])
     }
   }
-  *v = fv
+  *r = fr
+}
+
+// this effectively removes release candidates as well as pending releases
+func filterPrerelease(r *[]*github.RepositoryRelease) {
+  var fr []*github.RepositoryRelease
+  releases := *r
+  for i := 0; i < len(releases); i++ {
+    if !releases[i].GetPrerelease() {
+      fr = append(fr, releases[i])
+    }
+  }
+  *r = fr
 }
 
-func filterPrerelease(r []*github.RepositoryRelease) []string {
+func getVersionsFromReleases(r *[]*github.RepositoryRelease) []string {
   var versions []string
-  for _, release := range r {
-    version := release.GetTagName()
-    if !release.GetPrerelease() {
-      versions = append(versions, version)
+  releases := *r
+  for i := 0; i < len(releases); i++ {
+    versions = append(versions, *releases[i].TagName)
+  }
+  return versions
       // [
       //   "v1.28.14+rke2r1",
       //   "v1.30.1+rke2r3",
@@ -145,9 +141,6 @@ func filterPrerelease(r []*github.RepositoryRelease) []string {
       //   "v1.29.5+rke2r1",
       //   "v1.28.17+rke2r1",
       // ]
-    }
-  }
-  return versions
 }
 
 func sortVersions(v *[]string) {
@@ -467,3 +460,29 @@ func CheckReady(t *testing.T, kubeconfigPath string) {
   }
 	t.Logf("Ready script output: %s", out)
 }
+
+func CheckRunning(t *testing.T, kubeconfigPath string) {
+  repoRoot, err := filepath.Abs(g.GetRepoRoot(t))
+  if err != nil {
+    t.Logf("Error getting git root directory: %v", err)
+    t.Fail()
+  }
+  script, err := os.ReadFile(repoRoot + "/test/scripts/runningPods.sh")
+	if err != nil {
+		t.Logf("Error reading script: %v", err)
+    t.Fail()
+	}
+	readyScript := shell.Command{
+		Command: "bash",
+		Args:    []string{"-c", string(script)},
+		Env: map[string]string{
+			"KUBECONFIG": kubeconfigPath,
+		},
+	}
+	out, err := shell.RunCommandAndGetOutputE(t, readyScript)
+  if err != nil {
+    t.Logf("Error running script: %s", err)
+    t.Fail()
+  }
+	t.Logf("Ready script output: %s", out)
+}