Merge pull request ClickHouse#79089 from ClickHouse/ci_more_debugging_logs

CI: Add debug logs

Author: maxknv

ci/praktika/hook_html.py

@@ -142,9 +142,11 @@ def push_pending_ci_report(cls, _workflow):
                 # fetch running status with start_time for current job
                 result = Result.from_fs(job.name)
             else:
-                result = Result.generate_pending(job.name)
+                result = Result.create_new(job.name, Result.Status.PENDING)
             results.append(result)
-        summary_result = Result.generate_pending(_workflow.name, results=results)
+        summary_result = Result.create_new(
+            _workflow.name, Result.Status.RUNNING, results=results
+        )
         summary_result.start_time = Utils.timestamp()
         summary_result.links.append(env.CHANGE_URL)
         summary_result.links.append(env.RUN_URL)
@@ -200,18 +202,26 @@ def configure(cls, _workflow):
                         sha=cache_record.sha,
                         job_name=skipped_job,
                     )
-                    result = Result.generate_skipped(
-                        skipped_job, [report_link], "reused from cache"
+                    result = Result.create_new(
+                        skipped_job,
+                        Result.Status.SKIPPED,
+                        [report_link],
+                        "reused from cache",
                     )
                 else:
-                    result = Result.generate_skipped(
-                        skipped_job, info=filtered_job_and_reason[skipped_job]
+                    result = Result.create_new(
+                        skipped_job,
+                        Result.Status.SKIPPED,
+                        info=filtered_job_and_reason[skipped_job],
                     )
                 results.append(result)
             if results:
-                assert _ResultS3.update_workflow_results(
-                    _workflow.name, new_sub_results=results
-                )
+                assert (
+                    _ResultS3.update_workflow_results(
+                        _workflow.name, new_sub_results=results
+                    )
+                    is None
+                ), "Workflow status supposed to remain 'running'"
 
     @classmethod
     def pre_run(cls, _workflow, _job):

ci/praktika/parser.py

@@ -236,10 +236,6 @@ def parse(self):
                     assert (
                         False
                     ), f"Artifact [{artifact_name}] has unsupported type [{artifact.type}]"
-            if not artifact.required_by and artifact.type != Artifact.Type.PHONY:
-                print(
-                    f"WARNING: Artifact [{artifact_name}] provided by job [{artifact.provided_by}] in workflow [{self.workflow_name}] has no job that requires it"
-                )
             if artifact.type == Artifact.Type.GH:
                 self.workflow_yaml_config.job_to_config[
                     artifact.provided_by

ci/praktika/result.py

@@ -344,23 +344,10 @@ def add_ext_key_value(self, key, value):
         return self
 
     @classmethod
-    def generate_pending(cls, name, results=None):
+    def create_new(cls, name, status, links=None, info="", results=None):
         return Result(
             name=name,
-            status=Result.Status.PENDING,
-            start_time=None,
-            duration=None,
-            results=results or [],
-            files=[],
-            links=[],
-            info="",
-        )
-
-    @classmethod
-    def generate_skipped(cls, name, links=None, info="", results=None):
-        return Result(
-            name=name,
-            status=Result.Status.SKIPPED,
+            status=status,
             start_time=None,
             duration=None,
             results=results or [],
@@ -511,10 +498,10 @@ def to_stdout_formatted(self, indent="", res=""):
 
 class ResultInfo:
     SETUP_ENV_JOB_FAILED = (
-        "Failed to set up job env, it's praktika bug or misconfiguration"
+        "Failed to set up job env, it is praktika bug or misconfiguration"
     )
     PRE_JOB_FAILED = (
-        "Failed to do a job pre-run step, it's praktika bug or misconfiguration"
+        "Failed to do a job pre-run step, it is praktika bug or misconfiguration"
     )
     KILLED = "Job killed or terminated, no Result provided"
     NOT_FOUND_IMPOSSIBLE = (
@@ -556,20 +543,21 @@ def copy_result_from_s3_with_version(cls, local_path):
         env = _Environment.get()
         file_name = Path(local_path).name
         local_dir = Path(local_path).parent
-        file_name_pattern = f"{file_name}_*"
-        for file_path in local_dir.glob(file_name_pattern):
-            file_path.unlink()
-        s3_path = f"{Settings.HTML_S3_PATH}/{env.get_s3_prefix()}/"
-        S3.copy_file_from_s3_matching_pattern(
-            s3_path=s3_path, local_path=local_dir, include=file_name_pattern
+        s3_path = f"{Settings.HTML_S3_PATH}/{env.get_s3_prefix()}"
+        latest_result_file = Shell.get_output(
+            f"aws s3 ls {s3_path}/{file_name}_ | awk '{{print $4}}' | sort -r | head -n 1",
+            strict=True,
+            verbose=True,
+        )
+        version = int(latest_result_file.split("_")[-1])
+        S3.copy_file_from_s3(
+            s3_path=f"{s3_path}/{latest_result_file}", local_path=local_dir
+        )
+        Shell.check(
+            f"cp {local_dir}/{latest_result_file} {local_path}",
+            strict=True,
+            verbose=True,
         )
-        result_files = []
-        for file_path in local_dir.glob(file_name_pattern):
-            result_files.append(file_path)
-        assert result_files, "No result files found"
-        result_files.sort()
-        version = int(result_files[-1].name.split("_")[-1])
-        Shell.check(f"cp {result_files[-1]} {local_path}", strict=True, verbose=True)
         return version
 
     @classmethod
@@ -698,6 +686,7 @@ def update_workflow_results(
             # when multiple concurrent jobs attempt to update the workflow report
             time.sleep(random.uniform(0, 2))
 
+        print(f"Workflow status changed: [{prev_status}] -> [{new_status}]")
         if prev_status != new_status:
             return new_status
         else:

ci/praktika/runner.py

@@ -73,7 +73,7 @@ def generate_local_run_environment(workflow, job, pr=None, sha=None):
 
         workflow_config.dump()
 
-        Result.generate_pending(job.name).dump()
+        Result.create_from(name=job.name, status=Result.Status.PENDING).dump()
 
     def _setup_env(self, _workflow, job):
         # source env file to write data into fs (workflow config json, workflow status json)
@@ -323,7 +323,7 @@ def _post_run(
                 info=info,
             ).dump()
         elif prerun_exit_code != 0:
-            info = f"ERROR: {ResultInfo.PRE_JOB_FAILED}"
+            info = ResultInfo.PRE_JOB_FAILED
             print(info)
             # set Result with error and logs
             Result(

ci/praktika/s3.py

@@ -72,7 +72,8 @@ def add_uploaded(cls, file_path):
         file_zize = cls.get_size_bytes(file_path)
         usage.uploaded += file_zize
         if file_name in usage.uploaded_details:
-            print(f"WARNING: Duplicated upload for filename [{file_name}]")
+            if not file_name.startswith("result_"):
+                print(f"WARNING: Duplicated upload for filename [{file_name}]")
             usage.uploaded_details[file_name] += file_zize
         else:
             usage.uploaded_details[file_name] = file_zize
@@ -109,7 +110,7 @@ def clean_s3_directory(cls, s3_path, include=""):
         cmd = f"aws s3 rm s3://{s3_path} --recursive"
         if include:
             cmd += f' --exclude "*" --include "{include}"'
-        cls.run_command_with_retries(cmd, retries=1)
+        cls.run_command_with_retries(cmd, retries=1, with_stderr=True)
         return
 
     @classmethod
@@ -196,7 +197,11 @@ def put(
 
     @classmethod
     def run_command_with_retries(
-        cls, command, retries=Settings.MAX_RETRIES_S3, no_strict=False
+        cls,
+        command,
+        retries=Settings.MAX_RETRIES_S3,
+        no_strict=False,
+        with_stderr=False,
     ):
         i = 0
         res = False
@@ -224,6 +229,8 @@ def run_command_with_retries(
                 print(
                     f"ERROR: aws s3 cp failed, stdout/stderr err: [{stderr}], out [{stdout}]"
                 )
+            elif with_stderr and (stdout or stderr):
+                print(f"stdout: {stdout}\nstderr: {stderr}")
             res = ret_code == 0
         if not res and not no_strict:
             raise RuntimeError(f"s3 command failed: [{stderr}]")
@@ -275,7 +282,7 @@ def copy_file_from_s3_matching_pattern(
         ).is_dir(), f"Path [{local_path}] does not exist or not a directory"
         assert s3_path.endswith("/"), f"s3 path is invalid [{s3_path}]"
         cmd = f'aws s3 cp s3://{s3_path}  {local_path} --exclude "{exclude}" --include "{include}" --recursive'
-        res = cls.run_command_with_retries(cmd, no_strict=no_strict)
+        res = cls.run_command_with_retries(cmd, no_strict=no_strict, with_stderr=True)
         if res:
             print(
                 "TODO: support StorageUsage.add_downloaded with matching pattern download"