Merge pull request #4456 from bruntib/verbatim_args

[feat] Introduce cc-verbatim-args-file

Author: dkrupp

analyzer/codechecker_analyzer/analyzers/clangsa/analyzer.py

@@ -12,11 +12,11 @@
 import os
 import plistlib
 import re
-import shlex
 import subprocess
+import sys
+from typing import List, Tuple
 
-from typing import List
-
+from codechecker_common import util
 from codechecker_common.logger import get_logger
 
 from codechecker_analyzer import analyzer_context, env
@@ -42,7 +42,7 @@
 def parse_clang_help_page(
     command: List[str],
     start_label: str
-) -> List[str]:
+) -> List[Tuple[str, str]]:
     """
     Parse the clang help page starting from a specific label.
     Returns a list of (flag, description) tuples.
@@ -121,6 +121,12 @@ class ClangSA(analyzer_base.SourceAnalyzer):
 
     __ctu_autodetection = None
 
+    __additional_analyzer_config = {
+        'cc-verbatim-args-file':
+            'A file path containing flags that are forwarded verbatim to the '
+            'analyzer tool. E.g.: cc-verbatim-args-file=<filepath>'
+    }
+
     def __init__(self, cfg_handler, buildaction):
         super().__init__(cfg_handler, buildaction)
         self.__disable_ctu = False
@@ -308,15 +314,17 @@ def get_checker_config(cls) -> List[str]:
         return parse_clang_help_page(command, 'OPTIONS:')
 
     @classmethod
-    def get_analyzer_config(cls) -> List[str]:
+    def get_analyzer_config(cls) -> List[Tuple[str, str]]:
         """Return the list of analyzer config options."""
         command = [cls.analyzer_binary(), "-cc1"]
 
         cls.__add_plugin_load_flags(command)
 
         command.append("-analyzer-config-help")
 
-        return parse_clang_help_page(command, 'OPTIONS:')
+        native_config = parse_clang_help_page(command, 'OPTIONS:')
+
+        return native_config + list(cls.__additional_analyzer_config.items())
 
     def post_analyze(self, result_handler):
         """
@@ -636,21 +644,6 @@ def construct_config_handler(cls, args):
                 'ctu_ast_mode' in args and \
                 args.ctu_ast_mode == 'parse-on-demand'
 
-        try:
-            with open(args.clangsa_args_cfg_file, 'r', encoding='utf8',
-                      errors='ignore') as sa_cfg:
-                handler.analyzer_extra_arguments = \
-                    re.sub(r'\$\((.*?)\)',
-                           env.replace_env_var(args.clangsa_args_cfg_file),
-                           sa_cfg.read().strip())
-                handler.analyzer_extra_arguments = \
-                    shlex.split(handler.analyzer_extra_arguments)
-        except IOError as ioerr:
-            LOG.debug_analyzer(ioerr)
-        except AttributeError as aerr:
-            # No clangsa arguments file was given in the command line.
-            LOG.debug_analyzer(aerr)
-
         checkers = ClangSA.get_analyzer_checkers()
 
         try:
@@ -716,7 +709,19 @@ def construct_config_handler(cls, args):
         if 'analyzer_config' in args and \
                 isinstance(args.analyzer_config, list):
             for cfg in args.analyzer_config:
-                if cfg.analyzer == cls.ANALYZER_NAME:
+                # TODO: The analyzer plugin should get only its own analyzer
+                # config options from outside.
+                if cfg.analyzer != cls.ANALYZER_NAME:
+                    continue
+
+                if cfg.option == 'cc-verbatim-args-file':
+                    try:
+                        handler.analyzer_extra_arguments = \
+                            util.load_args_from_file(cfg.value)
+                    except FileNotFoundError:
+                        LOG.error(f"File not found: {cfg.value}")
+                        sys.exit(1)
+                else:
                     handler.checker_config.append(f"{cfg.option}={cfg.value}")
 
         return handler

analyzer/codechecker_analyzer/analyzers/clangtidy/analyzer.py

@@ -15,13 +15,14 @@
 import os
 from pathlib import Path
 import re
-import shlex
 import shutil
 import subprocess
+import sys
 from typing import Iterable, List, Set, Tuple
 
 import yaml
 
+from codechecker_common import util
 from codechecker_common.logger import get_logger
 
 from codechecker_analyzer import analyzer_context, env
@@ -250,6 +251,12 @@ class ClangTidy(analyzer_base.SourceAnalyzer):
     # Cache object for get_analyzer_checkers().
     __analyzer_checkers = None
 
+    __additional_analyzer_config = {
+        'cc-verbatim-args-file':
+            'A file path containing flags that are forwarded verbatim to the '
+            'analyzer tool. E.g.: cc-verbatim-args-file=<filepath>'
+    }
+
     @classmethod
     def analyzer_binary(cls):
         return analyzer_context.get_context() \
@@ -348,9 +355,11 @@ def get_analyzer_config(cls):
                 universal_newlines=True,
                 encoding="utf-8",
                 errors="ignore")
-            return parse_analyzer_config(result)
+            native_config = parse_analyzer_config(result)
         except (subprocess.CalledProcessError, OSError):
-            return []
+            native_config = []
+
+        return native_config + list(cls.__additional_analyzer_config.items())
 
     def get_checker_list(self, config) -> Tuple[List[str], List[str]]:
         """
@@ -623,32 +632,26 @@ def construct_config_handler(cls, args):
             'add_gcc_include_dirs_with_isystem' in args and \
             args.add_gcc_include_dirs_with_isystem
 
-        # FIXME We cannot get the resource dir from the clang-tidy binary,
-        # therefore we get a sibling clang binary which of clang-tidy.
-        # TODO Support "clang-tidy -print-resource-dir" .
-        try:
-            with open(args.tidy_args_cfg_file, 'r', encoding='utf-8',
-                      errors='ignore') as tidy_cfg:
-                handler.analyzer_extra_arguments = \
-                    re.sub(r'\$\((.*?)\)',
-                           env.replace_env_var(args.tidy_args_cfg_file),
-                           tidy_cfg.read().strip())
-                handler.analyzer_extra_arguments = \
-                    shlex.split(handler.analyzer_extra_arguments)
-        except IOError as ioerr:
-            LOG.debug_analyzer(ioerr)
-        except AttributeError as aerr:
-            # No clang tidy arguments file was given in the command line.
-            LOG.debug_analyzer(aerr)
-
         analyzer_config = {}
         # TODO: This extra "isinstance" check is needed for
         # CodeChecker analyzers --analyzer-config. This command also
         # runs this function in order to construct a config handler.
         if 'analyzer_config' in args and \
                 isinstance(args.analyzer_config, list):
             for cfg in args.analyzer_config:
-                if cfg.analyzer == cls.ANALYZER_NAME:
+                # TODO: The analyzer plugin should get only its own analyzer
+                # config options from outside.
+                if cfg.analyzer != cls.ANALYZER_NAME:
+                    continue
+
+                if cfg.option == 'cc-verbatim-args-file':
+                    try:
+                        handler.analyzer_extra_arguments = \
+                            util.load_args_from_file(cfg.value)
+                    except FileNotFoundError:
+                        LOG.error(f"File not found: {cfg.value}")
+                        sys.exit(1)
+                else:
                     analyzer_config[cfg.option] = cfg.value
 
         # Reports in headers are hidden by default in clang-tidy. Re-enable it

analyzer/codechecker_analyzer/analyzers/cppcheck/analyzer.py

@@ -10,19 +10,19 @@
 """
 
 from collections import defaultdict
+import sys
 from packaging.version import Version
 from pathlib import Path
 import os
 import re
-import shlex
 import shutil
 import subprocess
 import xml.etree.ElementTree as ET
 
 from codechecker_common.logger import get_logger
 from codechecker_common import util
 
-from codechecker_analyzer import analyzer_context, env
+from codechecker_analyzer import analyzer_context
 from codechecker_analyzer.env import get_binary_in_path
 
 from .. import analyzer_base
@@ -303,11 +303,15 @@ def get_analyzer_config(cls):
         """
         Config options for cppcheck.
         """
-        return [("addons", "A list of cppcheck addon files."),
-                ("libraries", "A list of cppcheck library definiton files."),
-                ("platform", "The platform configuration .xml file."),
-                ("inconclusive", "Enable inconclusive reports.")
-                ]
+        return [
+            ("addons", "A list of cppcheck addon files."),
+            ("libraries", "A list of cppcheck library definiton files."),
+            ("platform", "The platform configuration .xml file."),
+            ("inconclusive", "Enable inconclusive reports."),
+            ("cc-verbatim-args-file",
+             "A file path containing flags that are forwarded verbatim to the "
+             "analyzer tool. E.g.: cc-verbatim-args-file=<filepath>")
+        ]
 
     @classmethod
     def get_checker_config(cls):
@@ -410,20 +414,17 @@ def construct_config_handler(cls, args):
 
         handler.analyzer_config = analyzer_config
 
-        try:
-            with open(args.cppcheck_args_cfg_file, 'r', encoding='utf8',
-                      errors='ignore') as sa_cfg:
-                handler.analyzer_extra_arguments = \
-                    re.sub(r'\$\((.*?)\)',
-                           env.replace_env_var(args.cppcheck_args_cfg_file),
-                           sa_cfg.read().strip())
-                handler.analyzer_extra_arguments = \
-                    shlex.split(handler.analyzer_extra_arguments)
-        except IOError as ioerr:
-            LOG.debug_analyzer(ioerr)
-        except AttributeError as aerr:
-            # No cppcheck arguments file was given in the command line.
-            LOG.debug_analyzer(aerr)
+        if 'analyzer_config' in args and \
+                isinstance(args.analyzer_config, list):
+            for cfg in args.analyzer_config:
+                if cfg.analyzer == cls.ANALYZER_NAME and \
+                        cfg.option == 'cc-verbatim-args-file':
+                    try:
+                        handler.analyzer_extra_arguments = \
+                            util.load_args_from_file(cfg.value)
+                    except FileNotFoundError:
+                        LOG.error(f"File not found: {cfg.value}")
+                        sys.exit(1)
 
         checkers = cls.get_analyzer_checkers()
 

analyzer/codechecker_analyzer/analyzers/gcc/analyzer.py

@@ -5,12 +5,13 @@
 #  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 #
 # -------------------------------------------------------------------------
-from collections import defaultdict
 from packaging.version import Version
 import shlex
 import subprocess
+import sys
 
 from codechecker_common.logger import get_logger
+from codechecker_common import util
 
 from codechecker_analyzer import analyzer_context
 
@@ -80,6 +81,8 @@ def construct_analyzer_cmd(self, result_handler):
         if not has_flag('-x', analyzer_cmd):
             analyzer_cmd.extend(['-x', compile_lang])
 
+        analyzer_cmd.extend(config.analyzer_extra_arguments)
+
         analyzer_cmd.append(self.source_file)
 
         LOG.debug_analyzer("Running analysis command "
@@ -126,7 +129,11 @@ def get_analyzer_config(cls):
         Config options for gcc.
         """
         # TODO
-        return []
+        return [
+            ('cc-verbatim-args-file',
+             'A file path containing flags that are forwarded verbatim to the '
+             'analyzer tool. E.g.: cc-verbatim-args-file=<filepath>')
+        ]
 
     @classmethod
     def get_checker_config(cls):
@@ -216,15 +223,21 @@ def construct_result_handler(self, buildaction, report_output,
     def construct_config_handler(cls, args):
         handler = GccConfigHandler()
 
-        analyzer_config = defaultdict(list)
-
         if 'analyzer_config' in args and \
                 isinstance(args.analyzer_config, list):
             for cfg in args.analyzer_config:
-                if cfg.analyzer == cls.ANALYZER_NAME:
-                    analyzer_config[cfg.option].append(cfg.value)
-
-        handler.analyzer_config = analyzer_config
+                # TODO: The analyzer plugin should get only its own analyzer
+                # config options from outside.
+                if cfg.analyzer != cls.ANALYZER_NAME:
+                    continue
+
+                if cfg.option == 'cc-verbatim-args-file':
+                    try:
+                        handler.analyzer_extra_arguments = \
+                            util.load_args_from_file(cfg.value)
+                    except FileNotFoundError:
+                        LOG.error(f"File not found: {cfg.value}")
+                        sys.exit(1)
 
         checkers = cls.get_analyzer_checkers()