continue with implementation of messages in result file

martin-mat · martin-mat · commit 551ead3dfdf7 · 2026-04-09T21:22:44.000+02:00
diff --git a/src/tasks/constants.cr b/src/tasks/constants.cr
@@ -9,11 +9,6 @@ CONFIG_FILE = "cnf-testsuite.yml"
 BASE_CONFIG = "./config.yml"
 COMMON_MANIFEST_FILE_PATH = "#{CNF_DIR}/common_manifest.yml"
 DEPLOYMENT_MANIFEST_FILE_NAME = "deployment_manifest.yml"
-PASSED = "passed"
-FAILED = "failed"
-SKIPPED = "skipped"
-NA = "na"
-ERROR = "error"
 # todo move to helm module
 # CHART_YAML = "Chart.yaml"
 DEFAULT_POINTSFILENAME = "points_v1.yml"
diff --git a/src/tasks/utils/cnf_manager/points.cr b/src/tasks/utils/cnf_manager/points.cr
@@ -9,27 +9,111 @@ module CNFManager
     Failed
     Skipped
     NA
-    Neutral
     Pass5
     Pass3
     Error
 
     def to_basic
       case self
-      when Pass5, Pass3
-        ret = CNFManager::ResultStatus::Passed
-      when Neutral
-        ret = CNFManager::ResultStatus::Failed
-      else
-        ret = self
+      when Pass5, Pass3 then CNFManager::ResultStatus::Passed
+      else self
+      end
+    end
+
+    def to_string
+      case self.to_basic
+      when Passed then "passed"
+      when Failed then "failed"
+      when Skipped then "skipped"
+      when NA the "na"
+      when Error then "error"
+      else ""
       end
     end
   end
 
-  struct TestCaseResult
-    property state, result_message
+  class TestCaseResult
+    property testcase : String
+    property status : CNFManager::ResultStatus
+    property result_message : String?
+    property result_description : Array(String)
+    property start_time : Time
+    property end_time : Time
+
+    def initialize(@testcase : String = "",
+                   @status : CNFManager::ResultStatus = CNFManager::ResultStatus::Skipped,
+                   @result_message : String? = nil,
+                   @result_description : Array(String) = [] of String)
+    end
+
+    def update(status : CNFManager::ResultStatus, message : String?)
+      @status = status
+      @result_message = message
+    end
+
+    def passed(message : String?)
+      @status = CNFManager::ResultStatus::Passed
+      @result_message = message
+    end
+
+    def failed(message : String?)
+      @status = CNFManager::ResultStatus::Failed
+      @result_message = message
+    end
+
+    def skipped(message : String?)
+      @status = CNFManager::ResultStatus::Skipped
+      @result_message = message
+    end
+
+    def error(message : String?)
+      @status = CNFManager::ResultStatus::Error
+      @result_message = message
+    end
+
+    def append_description(text : String)
+      @result_description << text
+    end
+
+    def set_start_time()
+      @start_time = Time.utc
+    end
+
+    def set_end_time()
+      @end_time = Time.utc
+    end
+
+    def duration() : Time::Span
+      @end_time - @start_time
+    end
 
-    def initialize(@state : CNFManager::ResultStatus, @result_message : String? = nil)
+    def points() : 
+
+    def set_testcase(testcase : String)
+      @testcase = testcase
+    end
+
+    def decorated_result_message() : String
+      tc_emoji = CNFManager::Points.emoji_by_task(@testcase)
+      cat_emoji = CNFManager::Points.task_emoji_by_task(@testcase)↲
+      case @status.to_basic↲
+      when CNFManager::ResultStatus::Passed↲
+        "   #{cat_emoji}PASSED: [#{@testcase}] #{@result_message} #{tc_emoji}"
+      when CNFManager::ResultStatus::Failed↲
+        "   #{cat_emoji}FAILED: [#{@testcase}] #{@result_message} #{tc_emoji}"
+      when CNFManager::ResultStatus::Skipped↲
+        "⏭️  #{cat_emoji}SKIPPED: [#{@testcase}] #{@result_message} #{tc_emoji}"
+      when CNFManager::ResultStatus::NA↲
+        "⏭️  #{cat_emoji}N/A: [#{@testcase}] #{@result_message} #{tc_emoji}"
+      when CNFManager::ResultStatus::Error↲
+        "💥  #{cat_emoji}ERROR: [#{@testcase}] #{@result_message}"
+      else:
+        ""
+      end
+    end
+
+    def self.empty
+      new
     end
   end
 
@@ -275,8 +359,8 @@ module CNFManager
       {max_points, max_passed}
     end
 
-    def self.upsert_task(task, status, points, start_time, message)
-      logger = @@logger.for("upsert_task-#{task}")
+    def self.upsert_task(result : CNFManager::TestCaseResult)
+      logger = @@logger.for("upsert_task-#{result.testcase}")
 
       # Raise exception when results file does not exists.
       CNFManager::Points::Results.ensure_results_file!
@@ -287,36 +371,22 @@ module CNFManager
       result_items = result_items.reject { |x| x["name"] == task }
 
       end_time = Time.utc
-      task_runtime = (end_time - start_time)
 
       # The task result info has to be appeneded to an array of YAML::Any
       # So encode it into YAML and parse it back again to assign it.
       #
       # Only add task timestamps if the env var is set.
-      if status == "passed"
-        task_result_info = {
-          name:         task,
-          status:       status,
-          type:         task_type_by_task(task),
-          points:       points,
-          start_time:   start_time,
-          end_time:     end_time,
-          task_runtime: "#{task_runtime}",
-        }
-        result_items << YAML.parse(task_result_info.to_yaml)
-      else 
-        task_result_info = {
-          name:                 task,
-          status:               status,
-          status_description:   message,
-          type:                 task_type_by_task(task),
-          points:               points,
-          start_time:           start_time,
-          end_time:             end_time,
-          task_runtime:         "#{task_runtime}",
-        }
-        result_items << YAML.parse(task_result_info.to_yaml)
-      end
+      task_result_info = {
+        name:         result.testcase,
+        status:       result.status.to_string,
+        status_description:   result.result_description,
+        type:         task_type_by_task(result.testcase),
+        points:       result.points,
+        start_time:   result.start_time,
+        end_time:     result.end_time,
+        task_runtime: "#{result.duration}",
+      }
+      result_items << YAML.parse(task_result_info.to_yaml)
 
       File.open("#{Results.file}", "w") do |f|
         YAML.dump({name:              results["name"],
@@ -327,24 +397,9 @@ module CNFManager
                    exit_code:         results["exit_code"],
                    items:             result_items}, f)
       end
-      logger.debug { "Task start time: #{start_time}, end time: #{end_time}" }
-      logger.info { "Task: '#{task}' has status: '#{status}' and is awarded: #{points} points." +
-        "Runtime: #{task_runtime}" }
-    end
-
-    def self.failed_task(task, msg)
-      upsert_task(task, FAILED, task_points(task, false), start_time, msg)
-      stdout_failure "#{msg}"
-    end
-
-    def self.passed_task(task, msg)
-      upsert_task(task, PASSED, task_points(task), start_time, msg)
-      stdout_success "#{msg}"
-    end
-
-    def self.skipped_task(task, msg)
-      upsert_task(task, SKIPPED, task_points(task), start_time, msg)
-      stdout_success "#{msg}"
+      logger.debug { "Task start time: #{result.start_time}, end time: #{result.end_time}" }
+      logger.info { "Test case: '#{result.testcase}' has status: '#{result.status}' and is awarded: #{points} points." +
+                    "Runtime: #{result.duration}" }
     end
 
     def self.failed_required_tasks
diff --git a/src/tasks/utils/cnf_manager/task.cr b/src/tasks/utils/cnf_manager/task.cr
@@ -22,7 +22,7 @@ module CNFManager
     end
 
     def self.task_runner(args, task : Sam::Task? = nil, check_cnf_installed = true,
-                         &block : (Sam::Args, CNFInstall::Config::Config) -> (
+                         &block : (Sam::Args, CNFInstall::Config::Config, CNFManager::TestCaseResult) -> (
                            String | Colorize::Object(String) | CNFManager::TestCaseResult?)
     )
       CNFManager::Points::Results.ensure_results_file!
@@ -36,7 +36,7 @@ module CNFManager
     end
 
     def self.all_cnfs_task_runner(args, task : Sam::Task? = nil,
-                                  &block : (Sam::Args, CNFInstall::Config::Config) -> (
+                                  &block : (Sam::Args, CNFInstall::Config::Config, CNFManager::TestCaseResult) -> (
                                     String | Colorize::Object(String) | CNFManager::TestCaseResult?)
     )
       cnf_configs = CNFManager.cnf_config_list(false)
@@ -55,68 +55,58 @@ module CNFManager
 
     # TODO give example for calling
     def self.single_task_runner(args, task : Sam::Task? = nil, 
-                                &block : (Sam::Args, CNFInstall::Config::Config) -> (
+                                &block : (Sam::Args, CNFInstall::Config::Config, CNFManager::TestCaseResult) -> (
                                   String | Colorize::Object(String) | CNFManager::TestCaseResult?)
     )
       logger = @@logger.for("task_runner")
       logger.debug { "Run task with args #{args.inspect}" }
 
-      begin
-        # platform tests don't have a cnf-config
-        if args.named["cnf-config"]?
-          config = CNFInstall::Config.parse_cnf_config_from_file(args.named["cnf-config"].as(String))
-        else
-          yaml_string = <<-YAML
-            config_version: v2
-            deployments:
-              helm_dirs:
-                - name: "platform-test-dummy-deployment"
-                  helm_directory: ""
-            YAML
-          config = CNFInstall::Config.parse_cnf_config_from_yaml(yaml_string)
-        end
+      # platform tests don't have a cnf-config
+      if args.named["cnf-config"]?
+        config = CNFInstall::Config.parse_cnf_config_from_file(args.named["cnf-config"].as(String))
+      else
+        yaml_string = <<-YAML
+          config_version: v2
+          deployments:
+            helm_dirs:
+              - name: "platform-test-dummy-deployment"
+                helm_directory: ""
+          YAML
+        config = CNFInstall::Config.parse_cnf_config_from_yaml(yaml_string)
+      end
 
-        test_start_time = Time.utc
-        if task
-          test_name = task.as(Sam::Task).name.as(String)
-          logger.for(test_name).info { "Starting test" }
-          stdout_info("🎬 Testing: [#{test_name}]")
-        end
+      result = CNFManager::TestCaseResult.empty
+      result.set_start_time()
+      if task
+        result.set_testcase(task.as(Sam::Task).name.as(String))
+        logger.for(result.test_name).info { "Starting test" }
+        stdout_info("🎬 Testing: [#{result.test_name}]")
+      end
 
-        ret = yield args, config
-        if ret.is_a?(CNFManager::TestCaseResult)
-          upsert_decorated_task(test_name, ret.state, ret.result_message, test_start_time)
-        end
-        # todo lax mode, never returns 1
-        if args.raw.includes? "strict"
-          if CNFManager::Points.failed_required_tasks.size > 0
-            logger.fatal { "Strict mode exception. Stopping execution." }
-            stdout_failure "Test Suite failed in strict mode. Stopping execution."
-            stdout_failure "Failed required tasks: #{CNFManager::Points.failed_required_tasks.inspect}"
-            update_yml("#{CNFManager::Points::Results.file}", "exit_code", "#{FAILURE_CODE}")
-            exit FAILURE_CODE
-          end
-        end
-        ret
+      begin
+        yield args, config, result
       rescue ex
-        # platform tests don't have a cnf-config
-        # Set exception key/value in results
-        # file to -1
-        test_start_time = Time.utc
+        result.error("Unexpected error occurred")
         logger.error { ex.message }
         ex.backtrace.each do |x|
           logger.error { x }
         end
+      ensure
+        result.set_end_time()
+        upsert_decorated_task(result)
+      end
 
-        update_yml("#{CNFManager::Points::Results.file}", "exit_code", "#{CRITICAL_FAILURE_CODE}")
-        if args.raw.includes? "strict"
+      # todo lax mode, never returns 1
+      if args.raw.includes? "strict"
+        if result.state in (CNFManager::ResultStatus::Error, CNFManager::ResultStatus::Failed)
           logger.fatal { "Strict mode exception. Stopping execution." }
           stdout_failure "Test Suite failed in strict mode. Stopping execution."
-          exit CRITICAL_FAILURE_CODE
-        else
-          logger.warn { "Exception caught, continue to the next task" }
-          upsert_decorated_task(test_name, CNFManager::ResultStatus::Error, 
-            "Unexpected error occurred", test_start_time)
+          if CNFManager::Points.failed_required_tasks.size > 0
+            stdout_failure "Failed required tasks: #{CNFManager::Points.failed_required_tasks.inspect}"
+          end
+          exit_code = result.state == CNFManager::ResultStatus::Error ? CRITICAL_FAILURE_CODE : FAILURE_CODE
+          update_yml("#{CNFManager::Points::Results.file}", "exit_code", "#{exit_code}")
+          exit exit_code
         end
       end
     end
diff --git a/src/tasks/utils/utils.cr b/src/tasks/utils/utils.cr
@@ -190,9 +190,9 @@ def update_yml(yml_file, top_level_key, value)
   end
 end
 
-def upsert_decorated_task(task, status : CNFManager::ResultStatus, message, start_time)
-  tc_emoji = CNFManager::Points.emoji_by_task(task)
-  cat_emoji = CNFManager::Points.task_emoji_by_task(task)
+def upsert_decorated_task(result : CNFManager::TestCaseResult)
+  tc_emoji = CNFManager::Points.emoji_by_task(result.testcase)
+  cat_emoji = CNFManager::Points.task_emoji_by_task(result.testcase)
   case status.to_basic
   when CNFManager::ResultStatus::Passed
     upsert_passed_task(task, "✔️  #{cat_emoji}PASSED: [#{task}] #{message} #{tc_emoji}", message, start_time)
diff --git a/src/tasks/workload/microservice.cr b/src/tasks/workload/microservice.cr
@@ -751,7 +751,7 @@ end
 
 desc "To check if the CNF uses a specialized init system"
 task "specialized_init_system", ["setup:install_cluster_tools"] do |t, args|
-  CNFManager::Task.task_runner(args, task: t) do |args, config|
+  CNFManager::Task.task_runner(args, task: t) do |args, config, result|
     error_occurred    = false
     resources_checked = false
 
@@ -790,7 +790,7 @@ task "specialized_init_system", ["setup:install_cluster_tools"] do |t, args|
 
         # Report failures
         results.each do |info|
-          stdout_failure("#{info.kind}/#{info.name} container '#{info.container}' uses non-specialized init '#{info.init_cmd}'")
+          result.append_description("#{info.kind}/#{info.name} has container '#{info.container}' with '#{info.init_cmd}' as init process")
         end
 
         # mark this resource as failing
@@ -799,13 +799,13 @@ task "specialized_init_system", ["setup:install_cluster_tools"] do |t, args|
     end
 
     if error_occurred
-      CNFManager::TestCaseResult.new(CNFManager::ResultStatus::Skipped, "An error occurred during container inspection")
+      result.skipped("An error occurred during container inspection")
     elsif !resources_checked
-      CNFManager::TestCaseResult.new(CNFManager::ResultStatus::Skipped, "Container checks not executed")
+      result.skipped("Container checks not executed")
     elsif !task_response
-      CNFManager::TestCaseResult.new(CNFManager::ResultStatus::Failed, "Containers do not use specialized init systems (ভ_ভ) ރ")
+      result.failed("Containers do not use specialized init systems (ভ_ভ) ރ")
     else
-      CNFManager::TestCaseResult.new(CNFManager::ResultStatus::Passed, "Containers use specialized init systems 🖥️")
+      result.passed("Containers use specialized init systems 🖥️")
     end
   end
-end
+end
