Skip to content

Commit 884459f

Browse files
committed
Refactor Cluster API setup and change logic of its testcase
- Refactor logs with usage of .for(), make them more informative - Add error handling - Clean the code - Change logic of testcase to look if cluster on which CNF is running was provisioned by Cluster API and not if it has Cluster API manager installed - Update spec for it - Run autoformatter Signed-off-by: Rafal Lal <rafal.lal@tietoevry.com>
1 parent 576477e commit 884459f

3 files changed

Lines changed: 148 additions & 127 deletions

File tree

spec/platform/cluster_api_spec.cr

Lines changed: 38 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,38 +1,49 @@
11
require "./../spec_helper"
2-
require "colorize"
32
require "file_utils"
43
require "../../src/tasks/utils/utils.cr"
54

6-
describe "Cluster API" do
7-
before_all do
8-
result = ShellCmd.run_testsuite("setup")
9-
result[:status].success?.should be_true
10-
end
11-
after_each do
12-
result = ShellCmd.run_testsuite("setup:cluster_api_uninstall")
13-
result[:status].success?.should be_true
14-
end
5+
describe "Platform" do
6+
describe "cluster_api_enabled", tags: ["cluster-api"] do
7+
before_all do
8+
result = ShellCmd.run_testsuite("setup")
9+
result[:status].success?.should be_true
10+
end
1511

16-
it "'clusterapi_enabled' should pass if cluster api is installed", tags: ["cluster-api"] do
17-
begin
18-
result = ShellCmd.run_testsuite("setup:cluster_api_install")
19-
current_dir = FileUtils.pwd
20-
FileUtils.cd("#{current_dir}")
21-
result = ShellCmd.run_testsuite("clusterapi_enabled poc")
22-
(/Cluster API is enabled/ =~ result[:output]).should_not be_nil
12+
# (rafal-lal) TODO: decide how to proceed with Cluster API test case
13+
# after_each do
14+
# result = ShellCmd.run_testsuite("uninstall_cluster_api")
15+
# result[:status].success?.should be_true
16+
# end
17+
18+
it "should pass if nodes are managed by Cluster API" do
19+
begin
20+
node_name = KubectlClient::Get.resource("nodes").dig("items").as_a.first.dig("metadata", "name").as_s
21+
KubectlClient::Utils.annotate("node", node_name, ["cluster.x-k8s.io/owner-name=test-cluster-quickstart",
22+
"cluster.x-k8s.io/machine=test-1",
23+
"cluster.x-k8s.io/cluster-name=test-cluster"])
24+
rescue ex : Exception
25+
ex.message.should be_nil
26+
end
27+
result = ShellCmd.run_testsuite("cluster_api_enabled poc")
28+
(/At least one node in the cluster is managed by Cluster API/ =~ result[:output]).should_not be_nil
2329
ensure
24-
Log.info { "Running Cleanup" }
25-
result = ShellCmd.run_testsuite("setup:cluster_api_uninstall")
30+
# (rafal-lal) TODO: decide how to proceed with Cluster API test case
31+
# result = ShellCmd.run_testsuite("uninstall_cluster_api")
32+
begin
33+
node_name = KubectlClient::Get.resource("nodes").dig("items").as_a.first.dig("metadata", "name").as_s
34+
KubectlClient::Utils.annotate("node", node_name, ["cluster.x-k8s.io/owner-name-",
35+
"cluster.x-k8s.io/machine-",
36+
"cluster.x-k8s.io/cluster-name-"])
37+
rescue Exception
38+
end
2639
end
27-
end
28-
29-
it "'clusterapi_enabled' should fail if cluster api is not installed", tags: ["cluster-api-fail"] do
30-
begin
31-
result = ShellCmd.run_testsuite("clusterapi_enabled poc")
32-
(/Cluster API NOT enabled/ =~ result[:output]).should_not be_nil
40+
41+
it "should fail if nodes are not managed by Cluster API" do
42+
result = ShellCmd.run_testsuite("cluster_api_enabled poc")
43+
(/No nodes in the cluster are managed by Cluster API/ =~ result[:output]).should_not be_nil
3344
ensure
34-
Log.info { "Running Cleanup" }
35-
result = ShellCmd.run_testsuite("setup:cluster_api_uninstall")
45+
# (rafal-lal) TODO: decide how to proceed with Cluster API test case
46+
# result = ShellCmd.run_testsuite("uninstall_cluster_api")
3647
end
3748
end
3849
end

src/tasks/platform/platform.cr

Lines changed: 32 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -83,52 +83,47 @@ task "k8s_conformance" do |t, args|
8383
end
8484
end
8585

86-
desc "Is Cluster Api available and managing a cluster?"
87-
task "clusterapi_enabled" do |t, args|
86+
desc "Is Cluster managed by Cluster API"
87+
task "cluster_api_enabled" do |t, args|
88+
logger = PLOG.for("cluster_api_enabled")
89+
logger.info { "Testing if cluster has nodes managed by Cluster API" }
90+
8891
CNFManager::Task.task_runner(args, task: t, check_cnf_installed: false) do
92+
# (rafal-lal) TODO: should this still be POC after discussing task
8993
unless check_poc(args)
90-
next CNFManager::TestCaseResult.new(CNFManager::ResultStatus::Skipped, "Cluster API not in poc mode")
94+
next CNFManager::TestCaseResult.new(CNFManager::ResultStatus::Skipped, "Cluster API not in POC mode")
9195
end
9296

93-
Log.debug { "clusterapi_enabled" }
94-
Log.info { "clusterapi_enabled args #{args.inspect}" }
95-
96-
# We test that the namespaces for cluster resources exist by looking for labels
97-
# I found those by running
98-
# clusterctl init
99-
# kubectl -n capi-system describe deployments.apps capi-controller-manager
100-
# https://cluster-api.sigs.k8s.io/clusterctl/commands/init.html#additional-information
101-
102-
# this indicates that cluster-api is installed
103-
clusterapi_namespaces_json = KubectlClient::Get.resource("namespace", selector: "clusterctl.cluster.x-k8s.io")
104-
Log.info { "clusterapi_namespaces_json: #{clusterapi_namespaces_json}" }
105-
106-
# check that a node is actually being manageed
107-
# TODO: suppress msg in the case that this resource does-not-exist which is what happens when cluster-api is not installed
108-
cmd = "kubectl get kubeadmcontrolplanes.controlplane.cluster.x-k8s.io -o json"
109-
Process.run(
110-
cmd,
111-
shell: true,
112-
output: clusterapi_control_planes_output = IO::Memory.new,
113-
error: stderr = IO::Memory.new
114-
)
97+
begin
98+
nodes = KubectlClient::Get.resource("nodes").dig("items").as_a
99+
rescue ex : KubectlClient::ShellCMD::K8sClientCMDException
100+
logger.error { "Error while getting cluster nodes: #{ex.message}" }
101+
next
102+
end
115103

116-
proc_clusterapi_control_planes_json = -> do
117-
begin
118-
JSON.parse(clusterapi_control_planes_output.to_s)
119-
rescue JSON::ParseException
120-
# resource does-not-exist rescue to empty json
121-
JSON.parse("{}")
104+
# Check if any of the cluster nodes have Cluster API annotations
105+
capi_nodes = [] of String
106+
capi_annotation_found = nodes.any? do |node|
107+
annotations = node.dig?("metadata", "annotations")
108+
if annotations.nil?
109+
false
110+
else
111+
if !annotations.dig?("cluster.x-k8s.io/owner-name").nil? &&
112+
!annotations.dig?("cluster.x-k8s.io/machine").nil? &&
113+
!annotations.dig?("cluster.x-k8s.io/cluster-name").nil?
114+
capi_nodes << node.dig("metadata", "name").as_s
115+
true
116+
end
122117
end
123118
end
119+
logger.info { "#{capi_nodes.size} out of #{nodes.size} are managed by Cluster API" }
124120

125-
clusterapi_control_planes_json = proc_clusterapi_control_planes_json.call
126-
Log.info { "clusterapi_control_planes_json: #{clusterapi_control_planes_json}" }
127-
128-
if clusterapi_namespaces_json["items"]? && clusterapi_namespaces_json["items"].as_a.size > 0 && clusterapi_control_planes_json["items"]? && clusterapi_control_planes_json["items"].as_a.size > 0
129-
CNFManager::TestCaseResult.new(CNFManager::ResultStatus::Passed, "Cluster API is enabled")
121+
if capi_annotation_found
122+
CNFManager::TestCaseResult.new(CNFManager::ResultStatus::Passed,
123+
"At least one node in the cluster is managed by Cluster API")
130124
else
131-
CNFManager::TestCaseResult.new(CNFManager::ResultStatus::Failed, "Cluster API NOT enabled")
125+
CNFManager::TestCaseResult.new(CNFManager::ResultStatus::Failed,
126+
"No nodes in the cluster are managed by Cluster API")
132127
end
133128
end
134129
end
Lines changed: 78 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -1,85 +1,100 @@
11
require "sam"
22
require "file_utils"
3-
require "colorize"
4-
require "totem"
53
require "http/client"
6-
require "halite"
74
require "../utils/utils.cr"
8-
require "json"
9-
require "yaml"
105

116
namespace "setup" do
127
desc "Install Cluster API for Kind"
13-
task "cluster_api_install" do |_, args|
14-
current_dir = FileUtils.pwd
8+
task "install_cluster_api" do |_, args|
9+
logger = SLOG.for("install_cluster_api")
10+
logger.info { "Installing Cluster API tool" }
11+
failed_msg = "Task 'install_cluster_api' failed"
1512

16-
download(Setup::CLUSTER_API_URL, Setup::CLUSTERCTL_BINARY)
17-
18-
Process.run(
19-
"sudo chmod +x ./clusterctl",
20-
shell: true,
21-
output: stdout = IO::Memory.new,
22-
error: stderr = IO::Memory.new
23-
)
24-
Process.run(
25-
"sudo mv ./clusterctl /usr/local/bin/clusterctl",
26-
shell: true,
27-
output: stdout = IO::Memory.new,
28-
error: stderr = IO::Memory.new
29-
)
30-
31-
Log.info { "Completed downloading clusterctl" }
32-
33-
clusterctl = Path["~/.cluster-api"].expand(home: true)
34-
35-
FileUtils.mkdir_p("#{clusterctl}")
13+
if Dir.exists?(Setup::CLUSTER_API_DIR)
14+
logger.notice { "cluster api directory: '#{Setup::CLUSTER_API_DIR}' already exists, clusterctl should be available" }
15+
next
16+
end
3617

37-
File.write("#{clusterctl}/clusterctl.yaml", "CLUSTER_TOPOLOGY: \"true\"")
18+
FileUtils.mkdir_p(Setup::CLUSTER_API_DIR)
19+
begin
20+
download(Setup::CLUSTER_API_URL, Setup::CLUSTERCTL_BINARY)
21+
rescue ex : Exception
22+
logger.error { "Error while downloading clusterctl binary" }
23+
stdout_error(failed_msg)
24+
# (rafal-lal) TODO: SAM tasks error handling in setup, should we fail whole testsuite run / ignore
25+
# or something else? Applicable to all Setup tasks.
26+
next
27+
end
28+
logger.debug { "Downloaded clusterctl binary" }
3829

39-
cluster_init_cmd = "clusterctl init --infrastructure docker"
40-
stdout = IO::Memory.new
41-
Process.run(cluster_init_cmd, shell: true, output: stdout, error: stdout)
42-
Log.for("clusterctl init").info { stdout }
30+
resp = ShellCmd.run("chmod +x #{Setup::CLUSTERCTL_BINARY}")
31+
unless resp[:status].success?
32+
logger.error { "Error while making cluster api binary: '#{Setup::CLUSTERCTL_BINARY}' executable" }
33+
stdout_error(failed_msg)
34+
next
35+
end
4336

44-
create_cluster_file = "#{current_dir}/capi.yaml"
37+
File.write("#{Setup::CLUSTER_API_DIR}/clusterctl.yaml", "CLUSTER_TOPOLOGY: \"true\"")
38+
unless ShellCmd.run("#{Setup::CLUSTERCTL_BINARY} init --infrastructure docker")[:status].success?
39+
logger.error { "Error while initializing Cluster API on the cluster" }
40+
stdout_error(failed_msg)
41+
end
4542

46-
create_cluster_cmd = "clusterctl generate cluster capi-quickstart --kubernetes-version v1.24.0 --control-plane-machine-count=3 --worker-machine-count=3 --flavor development > #{create_cluster_file} "
43+
cluster_name = "capi-quickstart"
44+
cluster_tpl_file = "#{Setup::CLUSTER_API_DIR}/capi.yaml"
45+
# (rafal-lal) TODO: add kubernetes version const to use widely in codebase
46+
generate_cmd = "#{Setup::CLUSTERCTL_BINARY} generate cluster #{cluster_name}" +
47+
"--kubernetes-version v1.32.0" +
48+
"--control-plane-machine-count=1" +
49+
"--worker-machine-count=1" +
50+
"--flavor development" +
51+
"--target-namespace #{DEFAULT_CNF_NAMESPACE}" +
52+
"> #{cluster_tpl_file}"
53+
unless ShellCmd.run(generate_cmd)[:status].success?
54+
logger.error { "Error while generating workload cluster YAML template" }
55+
stdout_error(failed_msg)
56+
end
4757

48-
Process.run(
49-
create_cluster_cmd,
50-
shell: true,
51-
output: create_cluster_stdout = IO::Memory.new,
52-
error: create_cluster_stderr = IO::Memory.new
53-
)
58+
is_ready = false
59+
begin
60+
KubectlClient::Apply.file(cluster_tpl_file)
61+
is_ready = KubectlClient::Wait.wait_for_resource_key_value("cluster", cluster_name,
62+
{"status", "phase"}, "Provisioned", 300, DEFAULT_CNF_NAMESPACE)
63+
rescue ex : KubectlClient::ShellCMD::K8sClientCMDException
64+
logger.error { "Error while waiting for cluster to be Ready: #{ex.message}" }
65+
stdout_error(failed_msg)
66+
end
5467

55-
# TODO (rafal-lal): Connection error is expected in first couple tries, but it's not
56-
# reasonable to rescue it inside 'wait_for_install_by_apply' method, hence the while
57-
# loop here. Ideally this should be implemented in different way so we don't have to
58-
# rescue NetworkError at all. 'loop_count' var added so testsuite won't hang
59-
# indefinitely here.
60-
loop_break = false
61-
loop_count = 0
62-
while !loop_break && loop_count < 10
63-
begin
64-
KubectlClient::Wait.wait_for_install_by_apply(create_cluster_file)
65-
loop_break = true
66-
rescue KubectlClient::ShellCMD::NetworkError
67-
sleep 3.seconds
68-
loop_count += 1
69-
end
68+
unless is_ready
69+
logger.error { "Manifest apply not succesful or timed out while waiting for cluster to be Ready" }
70+
stdout_error(failed_msg)
71+
next
7072
end
7173

72-
Log.for("clusterctl-create").info { create_cluster_stdout.to_s }
73-
Log.info { "cluster api setup complete" }
74+
logger.info { "Cluster API provisioned cluster '#{cluster_name}' is ready to use" }
7475
end
7576

7677
desc "Uninstall Cluster API"
77-
task "cluster_api_uninstall" do |_, args|
78-
current_dir = FileUtils.pwd
79-
delete_cluster_file = "#{current_dir}/capi.yaml"
80-
begin KubectlClient::Delete.file("#{delete_cluster_file}") rescue KubectlClient::ShellCMD::NotFoundError end
78+
task "uninstall_cluster_api" do |_, args|
79+
logger = SLOG.for("uninstall_cluster_api")
80+
logger.info { "Uninstalling Cluster API tool" }
81+
82+
begin
83+
KubectlClient::Delete.file("#{Setup::CLUSTER_API_DIR}/capi.yaml")
84+
rescue KubectlClient::ShellCMD::NotFoundError
85+
logger.debug { "Cluster API 'cluster' resource does not exists" }
86+
rescue ex : KubectlClient::ShellCMD::K8sClientCMDException
87+
logger.error { "Error while deleting Cluster API 'cluster' resource: #{ex.message}" }
88+
stdout_error("Error while deleting Cluster API 'cluster' resource. Check logs for more info.")
89+
end
90+
91+
response = ShellCmd.run("#{Setup::CLUSTERCTL_BINARY} delete --all --include-crd --include-namespace")
92+
unless response[:status].success?
93+
logger.error { "Error while deleting Cluster API from the cluster: #{response[:error]}" }
94+
stdout_error("Error while deleting Cluster API 'cluster' resource. Check logs for more info.")
95+
next
96+
end
8197

82-
cmd = "clusterctl delete --all --include-crd --include-namespace"
83-
Process.run(cmd, shell: true, output: stdout = IO::Memory.new, error: stderr = IO::Memory.new)
98+
logger.info { "Cluster API uninstalled from the cluster" }
8499
end
85100
end

0 commit comments

Comments
 (0)