Merge pull request #96 from marcosps/fix-gcc-args-catching-issue

extract: Get gcc args by using compile_commands.json

Author: fgyanz

klpbuild/plugins/extractor.py

@@ -3,7 +3,7 @@
 # Copyright (C) 2021-2024 SUSE
 # Author: Marcos Paulo de Souza <[email protected]>
 
-import concurrent.futures
+from concurrent.futures import ThreadPoolExecutor
 import difflib as dl
 import json
 import logging
@@ -158,8 +158,7 @@ def get_make_cmd(out_dir, cs, filename, odir, sdir):
             elif shutil.which("gcc-7"):
                 cc = "gcc-7"
             else:
-                logging.error("Only gcc12 or higher are available, and it's problematic with kernel sources")
-                raise
+                raise RuntimeError("Only gcc12 or higher are available, and it's problematic with kernel sources")
 
             make_args = [
                 "make",
@@ -187,9 +186,8 @@ def get_make_cmd(out_dir, cs, filename, odir, sdir):
                 f.write(str(completed))
                 f.write("\n")
                 f.flush()
-            except subprocess.CalledProcessError as exc:
-                logging.error(f"Failed to run make for {cs.name()} ({cs.kernel}). Check file {str(log_path)} for more details.")
-                raise exc
+            except Exception:
+                raise RuntimeError(f"Failed to run make for {cs.name()} ({cs.kernel}). Check file {str(log_path)} for more details.")
 
             # 15.4 onwards changes the regex a little: -MD -> -MMD
             # 15.6 onwards we don't have -isystem.
@@ -210,9 +208,7 @@ def get_make_cmd(out_dir, cs, filename, odir, sdir):
                 f.flush()
 
             if not result:
-                logging.error(f"Failed to get the kernel cmdline for file {str(ofname)} in {cs.name()}. "
-                              f"Check file {str(log_path)} for more details.")
-                return None
+                raise RuntimeError(f"Failed to get the kernel cmdline for file {str(ofname)} in {cs.name()}. Check file {str(log_path)} for more details.")
 
             ret = Extractor.process_make_output(result.group(1))
 
@@ -230,11 +226,8 @@ def get_make_cmd(out_dir, cs, filename, odir, sdir):
 
             return ret
 
-        return None
-
-
-    # Generate the list of exported symbols
     def get_symbol_list(self, out_dir):
+        # Generate the list of exported symbols
         exts = []
 
         for ext_file in ["fun_exts", "obj_exts"]:
@@ -341,24 +334,25 @@ def apply_all_patches(self, cs, fil):
         if not patched:
             raise RuntimeError(f"{cs.name()}({cs.kernel}): Failed to apply patches. Aborting")
 
-
-
     def get_cmd_from_json(self, cs, fname):
-        cc_file = Path(cs.get_obj_dir(), "compile_commands.json")
-        # FIXME: compile_commands.json that is packaged with SLE/openSUSE
-        # doesn't quite work yet, so don't use it yet.
-        return None
+        cc_file = cs.get_obj_dir()/"compile_commands.json"
+
+        # Older codestreams doens't support compile_commands.json, so use make for them
+        if not cc_file.exists():
+            return None
 
         with open(cc_file) as f:
             buf = f.read()
         data = json.loads(buf)
         for d in data:
             if fname in d["file"]:
                 output = d["command"]
-                return Extractor.process_make_output(output)
+                # The arguments found on the file point to .., since they are generated
+                # when the kernel is compiled. Replace the .. by the codestream kernel source
+                # directory since klp-ccp needs to reach the files
+                return Extractor.process_make_output(output).replace("..", str(cs.get_src_dir()))
 
-        logging.error(f"Couldn't find cmdline for {fname}. Aborting")
-        return None
+        raise RuntimeError(f"Couldn't find cmdline for {fname} on {str(cc_file)}. Aborting")
 
     def cmd_args(self, cs, fname, out_dir, fdata, cmd):
         lp_out = Path(out_dir, cs.lp_out_file(self.lp_name, fname))
@@ -439,13 +433,10 @@ def process(self, args):
         # Make can regenerate fixdep for each file being processed per
         # codestream, so avoid the TXTBUSY error by serializing the 'make -sn'
         # calls. Make is pretty fast, so there isn't a real slow down here.
-        with self.make_lock:
-            cmd = self.get_cmd_from_json(cs, fname)
-            if not cmd:
-                cmd = Extractor.get_make_cmd(out_dir, cs, fname, odir, sdir)
-
+        cmd = self.get_cmd_from_json(cs, fname)
         if not cmd:
-            raise
+            with self.make_lock:
+                cmd = Extractor.get_make_cmd(out_dir, cs, fname, odir, sdir)
 
         args, lenv = self.cmd_args(cs, fname, out_dir, fdata, cmd)
 
@@ -462,9 +453,8 @@ def process(self, args):
             f.flush()
             try:
                 subprocess.run(args, cwd=odir, stdout=f, stderr=f, env=lenv, check=True)
-            except:
-                logging.error(f"Error when processing {cs.name()}:{fname}. Check file {out_log} for details.")
-                raise
+            except Exception as exc:
+                raise RuntimeError(f"Error when processing {cs.name()}:{fname}. Check file {out_log} for details.") from exc
 
         cs.files[fname]["ext_symbols"] = self.get_symbol_list(out_dir)
 
@@ -511,15 +501,14 @@ def run(self):
         logging.info(f"\nGenerating livepatches for {len(args)} file(s) using {self.workers} workers...")
         logging.info("\t\tCodestream\tFile")
 
-        with concurrent.futures.ThreadPoolExecutor(max_workers=self.workers) as executor:
-            results = executor.map(self.process, args)
+        with ThreadPoolExecutor(max_workers=self.workers) as executor:
             try:
-                for result in results:
-                    if result:
-                        logging.error(f"{cs}: {result}")
-            except:
-                executor.shutdown()
-                sys.exit(1)
+                futures = executor.map(self.process, args)
+                for future in futures:
+                    if future:
+                        logging.error(future)
+            except Exception as exc:
+                raise RuntimeError(str(exc)) from exc
 
         # Save the ext_symbols set by execute
         store_codestreams(self.lp_name, working_cs)

tests/test_extract.py

@@ -3,13 +3,17 @@
 # Copyright (C) 2021-2024 SUSE
 # Author: Marcos Paulo de Souza
 
+import json
 import inspect
 import logging
+import pytest
 
 from klpbuild.plugins.extractor import Extractor
 from klpbuild.plugins.setup import setup
+from klpbuild.klplib.codestreams_data import load_codestreams
 from klpbuild.klplib import utils
 
+import tests.utils as tests_utils
 
 def test_detect_file_without_ftrace_support(caplog):
     lp = "bsc_" + inspect.currentframe().f_code.co_name
@@ -29,8 +33,49 @@ def test_detect_file_without_ftrace_support(caplog):
     }
     setup(**setup_args)
 
-
     with caplog.at_level(logging.WARNING):
         Extractor(lp_name=lp, lp_filter=cs, apply_patches=False, avoid_ext=[]).run()
 
     assert "lib/seq_buf.o is not compiled with livepatch support (-pg flag)" in caplog.text
+
+
+def test_compile_commands_enoent():
+    """
+    Check if the extraction fails when a file isn't found on
+    compile_commands.json file
+    """
+
+    lp = "bsc_" + inspect.currentframe().f_code.co_name
+    cs = "15.6u8"
+
+    setup_args = {
+        "lp_name": lp,
+        "lp_filter": cs,
+        "no_check": False,
+        "archs": [utils.ARCH],
+        "cve": None,
+        "conf": "CONFIG_HID",
+        "module": "vmlinux",
+        "file_funcs": [["drivers/hid/hid-core.c", "hid_alloc_report_buf"]],
+        "mod_file_funcs": [],
+        "conf_mod_file_funcs": []
+    }
+    setup(**setup_args)
+
+    # rename the entry on files to a filename that doesn't exists (hid_core.c)
+    data = tests_utils.get_codestreams_file(lp)
+    file_funcs = data["codestreams"][cs]["files"].pop("drivers/hid/hid-core.c")
+    data["codestreams"][cs]["files"]["drivers/hid/hid_core.c"] = file_funcs
+
+    # write back the changed codestreams.json file
+    with open(utils.get_workdir(lp)/"codestreams.json", "r+") as f:
+        f.seek(0)
+        f.write(json.dumps(data, indent=4))
+        f.truncate()
+
+    # reload the codestreams after the change
+    load_codestreams(lp)
+
+    # Now it should fail with hid_core.c that doesn't exists on compile_commands.json
+    with pytest.raises(RuntimeError, match=r"Couldn't find cmdline for drivers/hid/hid_core.c on.*compile_commands.json. Aborting"):
+        Extractor(lp_name=lp, lp_filter=cs, apply_patches=False, avoid_ext=[]).run()