Updates in setup directory

- Fill setup/constants with consts with versions, urls and so on
- Fix build issues
- Refactor few baseline setup tasks (setup, cluster_setup, cnf_setup)
- Add SAM namespace called 'setup' to group setup tasks
- Remove unused cnf_conformance_setup.cr file

Signed-off-by: Rafal Lal <rafal.lal@tietoevry.com>

Author: rafal-lal

spec/workload/microservice_spec.cr

@@ -3,7 +3,6 @@ require "colorize"
 require "../../src/tasks/utils/utils.cr"
 require "kubectl_client"
 require "helm"
-require "../../src/tasks/setup/dockerd_setup.cr"
 require "file_utils"
 require "sam"
 

spec/workload/registry_spec.cr

@@ -3,7 +3,6 @@ require "colorize"
 require "../../src/tasks/utils/utils.cr"
 require "kubectl_client"
 require "helm"
-require "../../src/tasks/setup/dockerd_setup.cr"
 require "file_utils"
 require "sam"
 

src/tasks/constants.cr

@@ -17,10 +17,6 @@ ERROR = "error"
 # CHART_YAML = "Chart.yaml"
 DEFAULT_POINTSFILENAME = "points_v1.yml"
 SONOBUOY_K8S_VERSION = "0.56.14"
-KUBESCAPE_VERSION = "3.0.30"
-KUBESCAPE_FRAMEWORK_VERSION = "1.0.316"
-KIND_VERSION = "0.27.0"
-KIND_DOWNLOAD_URL = "https://github.com/kubernetes-sigs/kind/releases/download/v#{KIND_VERSION}/kind-linux-amd64"
 SONOBUOY_OS = "linux"
 IGNORED_SECRET_TYPES = ["kubernetes.io/service-account-token", "kubernetes.io/dockercfg", "kubernetes.io/dockerconfigjson", "helm.sh/release.v1"]
 EMPTY_JSON = JSON.parse(%({}))
@@ -29,6 +25,7 @@ SPECIALIZED_INIT_SYSTEMS = ["tini", "dumb-init", "s6-svscan"]
 ROLLING_VERSION_CHANGE_TEST_NAMES = ["rolling_update", "rolling_downgrade", "rolling_version_change"]
 WORKLOAD_RESOURCE_KIND_NAMES = ["replicaset", "deployment", "statefulset", "pod", "daemonset"]
 
+# (rafal-lal)T TODO: why do we need 2 namespaces, trim that to 1
 TESTSUITE_NAMESPACE = "cnf-testsuite"
 DEFAULT_CNF_NAMESPACE = "cnf-default"
 # (kosstennbl) Needed only for manifest deployments, where we don't have control over installation namespace

src/tasks/setup/cluster_api_setup.cr

@@ -4,80 +4,82 @@ require "colorize"
 require "totem"
 require "http/client"
 require "halite"
-require "./utils/utils.cr"
+require "../utils/utils.cr"
 require "json"
 require "yaml"
 
-desc "Install Cluster API for Kind"
-task "cluster_api_install" do |_, args|
-  current_dir = FileUtils.pwd
+namespace "setup" do
+  desc "Install Cluster API for Kind"
+  task "cluster_api_install" do |_, args|
+    current_dir = FileUtils.pwd
 
-  HttpHelper.download("https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.2.0/clusterctl-linux-amd64", "clusterctl")
+    download(Setup::CLUSTER_API_URL, Setup::CLUSTERCTL_BINARY)
 
-  Process.run(
-    "sudo chmod +x ./clusterctl",
-    shell: true,
-    output: stdout = IO::Memory.new,
-    error: stderr = IO::Memory.new
-  )
-  Process.run(
-    "sudo mv ./clusterctl /usr/local/bin/clusterctl",
-    shell: true,
-    output: stdout = IO::Memory.new,
-    error: stderr = IO::Memory.new
-  )
+    Process.run(
+      "sudo chmod +x ./clusterctl",
+      shell: true,
+      output: stdout = IO::Memory.new,
+      error: stderr = IO::Memory.new
+    )
+    Process.run(
+      "sudo mv ./clusterctl /usr/local/bin/clusterctl",
+      shell: true,
+      output: stdout = IO::Memory.new,
+      error: stderr = IO::Memory.new
+    )
 
-  Log.info { "Completed downloading clusterctl" }
+    Log.info { "Completed downloading clusterctl" }
 
-  clusterctl = Path["~/.cluster-api"].expand(home: true)
+    clusterctl = Path["~/.cluster-api"].expand(home: true)
 
-  FileUtils.mkdir_p("#{clusterctl}")
+    FileUtils.mkdir_p("#{clusterctl}")
 
-  File.write("#{clusterctl}/clusterctl.yaml", "CLUSTER_TOPOLOGY: \"true\"")
+    File.write("#{clusterctl}/clusterctl.yaml", "CLUSTER_TOPOLOGY: \"true\"")
 
-  cluster_init_cmd = "clusterctl init --infrastructure docker"
-  stdout = IO::Memory.new
-  Process.run(cluster_init_cmd, shell: true, output: stdout, error: stdout)
-  Log.for("clusterctl init").info { stdout }
+    cluster_init_cmd = "clusterctl init --infrastructure docker"
+    stdout = IO::Memory.new
+    Process.run(cluster_init_cmd, shell: true, output: stdout, error: stdout)
+    Log.for("clusterctl init").info { stdout }
 
-  create_cluster_file = "#{current_dir}/capi.yaml"
+    create_cluster_file = "#{current_dir}/capi.yaml"
 
-  create_cluster_cmd = "clusterctl generate cluster capi-quickstart   --kubernetes-version v1.24.0   --control-plane-machine-count=3 --worker-machine-count=3  --flavor development > #{create_cluster_file} "
+    create_cluster_cmd = "clusterctl generate cluster capi-quickstart   --kubernetes-version v1.24.0   --control-plane-machine-count=3 --worker-machine-count=3  --flavor development > #{create_cluster_file} "
 
-  Process.run(
-    create_cluster_cmd,
-    shell: true,
-    output: create_cluster_stdout = IO::Memory.new,
-    error: create_cluster_stderr = IO::Memory.new
-  )
+    Process.run(
+      create_cluster_cmd,
+      shell: true,
+      output: create_cluster_stdout = IO::Memory.new,
+      error: create_cluster_stderr = IO::Memory.new
+    )
 
-  # TODO (rafal-lal): Connection error is expected in first couple tries, but it's not
-  # reasonable to rescue it inside 'wait_for_install_by_apply' method, hence the while
-  # loop here. Ideally this should be implemented in different way so we don't have to
-  # rescue NetworkError at all. 'loop_count' var added so testsuite won't hang
-  # indefinitely here.
-  loop_break = false
-  loop_count = 0
-  while !loop_break && loop_count < 10
-    begin
-      KubectlClient::Wait.wait_for_install_by_apply(create_cluster_file)
-      loop_break = true
-    rescue KubectlClient::ShellCMD::NetworkError
-      sleep 3.seconds
-      loop_count += 1
+    # TODO (rafal-lal): Connection error is expected in first couple tries, but it's not
+    # reasonable to rescue it inside 'wait_for_install_by_apply' method, hence the while
+    # loop here. Ideally this should be implemented in different way so we don't have to
+    # rescue NetworkError at all. 'loop_count' var added so testsuite won't hang
+    # indefinitely here.
+    loop_break = false
+    loop_count = 0
+    while !loop_break && loop_count < 10
+      begin
+        KubectlClient::Wait.wait_for_install_by_apply(create_cluster_file)
+        loop_break = true
+      rescue KubectlClient::ShellCMD::NetworkError
+        sleep 3.seconds
+        loop_count += 1
+      end
     end
-  end
 
-  Log.for("clusterctl-create").info { create_cluster_stdout.to_s }
-  Log.info { "cluster api setup complete" }
-end
+    Log.for("clusterctl-create").info { create_cluster_stdout.to_s }
+    Log.info { "cluster api setup complete" }
+  end
 
-desc "Uninstall Cluster API"
-task "cluster_api_uninstall" do |_, args|
-  current_dir = FileUtils.pwd
-  delete_cluster_file = "#{current_dir}/capi.yaml"
-  begin KubectlClient::Delete.file("#{delete_cluster_file}") rescue KubectlClient::ShellCMD::NotFoundError end
+  desc "Uninstall Cluster API"
+  task "cluster_api_uninstall" do |_, args|
+    current_dir = FileUtils.pwd
+    delete_cluster_file = "#{current_dir}/capi.yaml"
+    begin KubectlClient::Delete.file("#{delete_cluster_file}") rescue KubectlClient::ShellCMD::NotFoundError end
 
-  cmd = "clusterctl delete --all --include-crd --include-namespace"
-  Process.run(cmd, shell: true, output: stdout = IO::Memory.new, error: stderr = IO::Memory.new)
+    cmd = "clusterctl delete --all --include-crd --include-namespace"
+    Process.run(cmd, shell: true, output: stdout = IO::Memory.new, error: stderr = IO::Memory.new)
+  end
 end

src/tasks/setup/cluster_setup.cr

@@ -1,30 +1,44 @@
 require "sam"
-require "file_utils"
-require "colorize"
 require "cluster_tools"
-require "totem"
-require "./utils/utils.cr"
 
-# CHAOS_MESH_VERSION = "v0.8.0"
+namespace "setup" do
+  desc "Install CNF Test Suite Cluster Tools"
+  task "install_cluster_tools" do |_, args|
+    logger = SLOG.for("install_cluster_tools")
+    logger.info { "Installing cluster_tools on the cluster" }
 
-desc "Install CNF Test Suite Cluster Tools"
-task "install_cluster_tools" do |_, args|
-  begin
-    ClusterTools.install
-  rescue e : ClusterTools::NamespaceDoesNotExistException 
-    Log.error { "#{e.message}" }
-    stdout_failure "Error: Namespace cnf-testsuite does not exist.\nPlease run 'cnf-testsuite setup' to create the necessary namespace."
-    exit(1)
+    begin
+      ClusterTools.install
+    rescue ex : ClusterTools::NamespaceDoesNotExistException
+      logger.error { "Error while installing: #{ex.message}" }
+      stdout_failure "Error: Namespace cnf-testsuite does not exist.\n" +
+                     "Please run 'cnf-testsuite setup' to create the necessary namespace."
+      exit(1)
+    rescue ex : Exception
+      logger.error { "Error while installing: #{ex.message}" }
+      stdout_failure "Unexpected error occured. Check logs for more info."
+      exit(1)
+    end
+    logger.info { "cluster_tools has been installed on the cluster" }
   end
-end
 
-desc "Uninstall CNF Test Suite Cluster Tools"
-task "uninstall_cluster_tools" do |_, args|
-  begin
-    ClusterTools.uninstall
-  rescue e : ClusterTools::NamespaceDoesNotExistException
-    Log.error { "#{e.message}" }
-    stdout_failure "Error: Namespace cnf-testsuite does not exist.\nPlease run 'cnf-testsuite setup' to create the necessary namespace."
-    exit(1)
+  desc "Uninstall CNF Test Suite Cluster Tools"
+  task "uninstall_cluster_tools" do |_, args|
+    logger = SLOG.for("uninstall_cluster_tools")
+    logger.info { "Uninstalling cluster_tools from the cluster" }
+
+    begin
+      ClusterTools.uninstall
+    rescue e : ClusterTools::NamespaceDoesNotExistException
+      logger.error { "Error while uninstalling: #{ex.message}" }
+      stdout_failure "Error: Namespace cnf-testsuite does not exist.\n" +
+                     "Please run 'cnf-testsuite setup' to create the necessary namespace."
+      exit(1)
+    rescue ex : Exception
+      logger.error { "Error while installing: #{ex.message}" }
+      stdout_failure "Unexpected error occured. Check logs for more info."
+      exit(1)
+    end
+    logger.info { "cluster_tools has been uninstalled on the cluster" }
   end
 end

src/tasks/setup/cnf_conformance_setup.cr

@@ -1,95 +1,34 @@
 require "sam"
-require "file_utils"
-require "colorize"
-require "totem"
-require "yaml"
-require "./utils/utils.cr"
+require "../utils/utils.cr"
 
-task "generate_config" do |_, args|
-  interactively_create_config()
-end
-
-task "cnf_install", ["helm_local_install", "create_namespace"] do |_, args|
-  if CNFManager.cnf_installed?
-    stdout_warning "A CNF is already installed. Installation of multiple CNFs is not allowed."
-    stdout_warning "To install a new CNF, uninstall the existing one by running: cnf_uninstall"
-    exit 0
-  end
-  if ClusterTools.install
-    stdout_success "ClusterTools installed"
-  else
-    stdout_failure "The ClusterTools installation timed out. Please check the status of the cluster-tools pods."
-    exit 1
-  end
-  stdout_success "CNF installation start."
-  CNFInstall.install_cnf(args)
-  stdout_success "CNF installation complete."
-end
-
-task "cnf_uninstall" do |_, args|
-  CNFInstall.uninstall_cnf()
-end
+namespace "setup" do
+  task "cnf_install", ["helm_local_install", "create_namespace"] do |_, args|
+    logger = SLOG.for("cnf_install")
+    logger.info { "Installing CNF to cluster" }
 
-def interactively_create_config
-  new_config = {
-    config_version: CNFInstall::Config::ConfigVersion::Latest.to_s,
-    deployments: {
-      helm_charts: [] of Hash(String, String),
-      helm_dirs: [] of Hash(String, String),
-      manifests: [] of Hash(String, String)
-    }
-  }
-
-  loop do
-    puts "Select deployment type:"
-    puts "1. Helm Chart"
-    puts "2. Helm Directory"
-    puts "3. Manifest Directory"
-    puts "4. Finish and save configuration"
-    choice = prompt("Enter your choice (1-4): ")
+    if CNFManager.cnf_installed?
+      stdout_warning "A CNF is already installed. Installation of multiple CNFs is not allowed."
+      stdout_warning "To install a new CNF, uninstall the existing one by running: cnf_uninstall"
+      exit 0
+    end
 
-    case choice
-    when "1"
-      helm_chart = {
-        "name" => prompt("Enter deployment name: "),
-        "helm_repo_name" => prompt("Enter Helm repository name: "),
-        "helm_repo_url" => prompt("Enter Helm repository URL: "),
-        "helm_chart_name" => prompt("Enter Helm chart name: ")
-      }
-      helm_chart["helm_chart_name"] = helm_chart["helm_chart_name"]
-      new_config[:deployments][:helm_charts] << helm_chart
-    when "2"
-      helm_dir = {
-        "name" => prompt("Enter deployment name: "),
-        "helm_directory" => prompt("Enter path to directory with Chart.yaml: ")
-      }
-      new_config[:deployments][:helm_dirs] << helm_dir
-    when "3"
-      manifest = {
-        "name" => prompt("Enter deployment name: "),
-        "manifest_directory" => prompt("Enter path to directory with manifest files: ")
-      }
-      new_config[:deployments][:manifests] << manifest
-    when "4"
-      break
+    if ClusterTools.install
+      stdout_success "ClusterTools installed"
     else
-      puts "Invalid choice. Please try again."
+      stdout_failure "The ClusterTools installation timed out. Please check the status of the cluster-tools pods."
+      exit 1
     end
-  end
 
-  yaml_config = new_config.to_yaml
-  puts "Generated Configuration:"
-  puts yaml_config
-
-  output_file = prompt("Choose output config path (leave empty for #{CONFIG_FILE}): ")
-  if output_file.strip.empty?
-    output_file = CONFIG_FILE
+    stdout_success "CNF installation start."
+    CNFInstall.install_cnf(args)
+    logger.info { "CNF installed successfuly" }
+    stdout_success "CNF installation ended."
   end
-  File.write(output_file, yaml_config)
-  puts "Configuration saved to #{output_file}"
-end
 
-def prompt(message)
-  print message
-  gets.try(&.strip) || ""
+  task "cnf_uninstall" do |_, args|
+    logger = SLOG.for("cnf_uninstall")
+    logger.info { "Uninstalling CNF from cluster" }
+    CNFInstall.uninstall_cnf
+    logger.info { "CNF uninstallation ended" }
+  end
 end