Merge branch 'release/v5.2.1'

Author: ivankravets

.github/workflows/examples.yml

@@ -7,7 +7,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-16.04, windows-latest, macos-latest]
+        os: [ubuntu-18.04, windows-latest, macos-latest]
         python-version: [3.7]
     runs-on: ${{ matrix.os }}
     steps:

.pylintrc

@@ -16,6 +16,7 @@ disable=
     useless-import-alias,
     bad-option-value,
     consider-using-dict-items,
+    consider-using-f-string,
 
     ; PY2 Compat
     super-with-arguments,

HISTORY.rst

@@ -8,6 +8,18 @@ PlatformIO Core 5
 
 **A professional collaborative platform for embedded development**
 
+5.2.1 (2021-10-11)
+~~~~~~~~~~~~~~~~~~
+
+- Clean a build environment and installed library dependencies using a new ``cleanall`` target (`issue #4062 <https://github.com/platformio/platformio-core/issues/4062>`_)
+- Override a default library builder via a new ``builder`` field in a ``build`` group of `library.json <https://docs.platformio.org/page/librarymanager/config.html#build>`__ manifest (`issue #3957 <https://github.com/platformio/platformio-core/issues/3957>`_)
+- Updated `Cppcheck <https://docs.platformio.org/page/plus/check-tools/cppcheck.html>`__ v2.6 with new checks, increased reliability of advanced addons (MISRA/CERT) and various improvements
+- Handle the "test" folder as a part of CLion project (`issue #4005 <https://github.com/platformio/platformio-core/issues/4005>`_)
+- Improved handling of a library root based on "Conan" or "CMake" build systems (`issue #3887 <https://github.com/platformio/platformio-core/issues/3887>`_)
+- Fixed a "KeyError: Invalid board option 'build.cpu'" when using a precompiled library with a board that does not have a CPU field in the manifest (`issue #4056 <https://github.com/platformio/platformio-core/issues/4056>`_)
+- Fixed a "FileExist" error when the `platformio ci <https://docs.platformio.org/en/latest/userguide/cmd_ci.html>`__ command is used in pair with the ``--keep-build-dir`` option (`issue #4011 <https://github.com/platformio/platformio-core/issues/4011>`_)
+- Fixed an issue with draft values of C++ language standards that broke static analysis via Cppcheck (`issue #3944 <https://github.com/platformio/platformio-core/issues/3944>`_)
+
 5.2.0 (2021-09-13)
 ~~~~~~~~~~~~~~~~~~
 

docs

@@ -14,7 +14,7 @@
 
 import sys
 
-VERSION = (5, 2, 0)
+VERSION = (5, 2, 1)
 __version__ = ".".join([str(s) for s in VERSION])
 
 __title__ = "platformio"
@@ -51,7 +51,7 @@
     "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor),
     "tool-unity": "~1.20500.0",
     "tool-scons": "~4.40200.0",
-    "tool-cppcheck": "~1.250.0",
+    "tool-cppcheck": "~1.260.0",
     "tool-clangtidy": "~1.120001.0",
     "tool-pvs-studio": "~7.14.0",
 }

platformio/__init__.py

@@ -67,9 +67,24 @@ def cli(ctx, force, caller, no_ansi):
     maintenance.on_platformio_start(ctx, force, caller)
 
 
-@cli.resultcallback()
-@click.pass_context
-def process_result(ctx, result, *_, **__):
+try:
+
+    @cli.result_callback()
+    @click.pass_context
+    def process_result(ctx, result, *_, **__):
+        _process_result(ctx, result)
+
+
+except (AttributeError, TypeError):  # legacy support for CLick > 8.0.1
+    print("legacy Click")
+
+    @cli.resultcallback()
+    @click.pass_context
+    def process_result(ctx, result, *_, **__):
+        _process_result(ctx, result)
+
+
+def _process_result(ctx, result):
     from platformio import maintenance
 
     maintenance.on_platformio_end(ctx, result)

platformio/__main__.py

@@ -149,10 +149,12 @@
     # pylint: disable=protected-access
     click._compat.isatty = lambda stream: True
 
-if env.GetOption("clean"):
-    env.PioClean(env.subst("$BUILD_DIR"))
+is_clean_all = "cleanall" in COMMAND_LINE_TARGETS
+if env.GetOption("clean") or is_clean_all:
+    env.PioClean(is_clean_all)
     env.Exit(0)
-elif not int(ARGUMENTS.get("PIOVERBOSE", 0)):
+
+if not int(ARGUMENTS.get("PIOVERBOSE", 0)):
     click.echo("Verbose mode can be enabled via `-v, --verbose` option")
 
 # Dynamically load dependent tools

platformio/builder/main.py

@@ -59,6 +59,16 @@ def new(env, path, verbose=int(ARGUMENTS.get("PIOVERBOSE", 0))):
                 clsname = "%sLibBuilder" % used_frameworks[0].title()
 
         obj = getattr(sys.modules[__name__], clsname)(env, path, verbose=verbose)
+
+        # Handle PlatformIOLibBuilder.manifest.build.builder
+        # pylint: disable=protected-access
+        if isinstance(obj, PlatformIOLibBuilder) and obj._manifest.get("build", {}).get(
+            "builder"
+        ):
+            obj = getattr(
+                sys.modules[__name__], obj._manifest.get("build", {}).get("builder")
+            )(env, path, verbose=verbose)
+
         assert isinstance(obj, LibBuilderBase)
         return obj
 
@@ -174,19 +184,19 @@ def src_filter(self):
 
     @property
     def include_dir(self):
-        if not all(
-            os.path.isdir(os.path.join(self.path, d)) for d in ("include", "src")
-        ):
-            return None
-        return os.path.join(self.path, "include")
+        for name in ("include", "Include"):
+            d = os.path.join(self.path, name)
+            if os.path.isdir(d):
+                return d
+        return None
 
     @property
     def src_dir(self):
-        return (
-            os.path.join(self.path, "src")
-            if os.path.isdir(os.path.join(self.path, "src"))
-            else self.path
-        )
+        for name in ("src", "Src"):
+            d = os.path.join(self.path, name)
+            if os.path.isdir(d):
+                return d
+        return self.path
 
     def get_include_dirs(self):
         items = []
@@ -491,6 +501,14 @@ def load_manifest(self):
             return {}
         return ManifestParserFactory.new_from_file(manifest_path).as_dict()
 
+    @property
+    def include_dir(self):
+        if not all(
+            os.path.isdir(os.path.join(self.path, d)) for d in ("include", "src")
+        ):
+            return None
+        return os.path.join(self.path, "include")
+
     def get_include_dirs(self):
         include_dirs = LibBuilderBase.get_include_dirs(self)
         if os.path.isdir(os.path.join(self.path, "src")):
@@ -566,9 +584,12 @@ def build_flags(self):
         if self._manifest.get("precompiled") in ("true", "full"):
             # add to LDPATH {build.mcu} folder
             board_config = self.env.BoardConfig()
-            self.env.PrependUnique(
-                LIBPATH=os.path.join(self.src_dir, board_config.get("build.cpu"))
-            )
+            for key in ("build.mcu", "build.cpu"):
+                libpath = os.path.join(self.src_dir, board_config.get(key, ""))
+                if not os.path.isdir(libpath):
+                    continue
+                self.env.PrependUnique(LIBPATH=libpath)
+                break
         ldflags = [flag for flag in ldflags if flag]  # remove empty
         return " ".join(ldflags) if ldflags else None
 
@@ -580,12 +601,6 @@ def load_manifest(self):
             return {}
         return ManifestParserFactory.new_from_file(manifest_path).as_dict()
 
-    @property
-    def include_dir(self):
-        if os.path.isdir(os.path.join(self.path, "include")):
-            return os.path.join(self.path, "include")
-        return None
-
     @property
     def src_dir(self):
         if os.path.isdir(os.path.join(self.path, "source")):

platformio/builder/tools/piolib.py

@@ -29,7 +29,7 @@ def VerboseAction(_, act, actstr):
     return Action(act, actstr)
 
 
-def PioClean(env, clean_dir):
+def PioClean(env, clean_all=False):
     def _relpath(path):
         if compat.IS_WINDOWS:
             prefix = os.getcwd()[:2].lower()
@@ -41,21 +41,30 @@ def _relpath(path):
                 return path
         return os.path.relpath(path)
 
-    if not os.path.isdir(clean_dir):
+    def _clean_dir(path):
+        clean_rel_path = _relpath(path)
+        for root, _, files in os.walk(path):
+            for f in files:
+                dst = os.path.join(root, f)
+                os.remove(dst)
+                print(
+                    "Removed %s"
+                    % (dst if not clean_rel_path.startswith(".") else _relpath(dst))
+                )
+
+    build_dir = env.subst("$BUILD_DIR")
+    libdeps_dir = env.subst("$PROJECT_LIBDEPS_DIR")
+    if os.path.isdir(build_dir):
+        _clean_dir(build_dir)
+        fs.rmtree(build_dir)
+    else:
         print("Build environment is clean")
-        env.Exit(0)
-    clean_rel_path = _relpath(clean_dir)
-    for root, _, files in os.walk(clean_dir):
-        for f in files:
-            dst = os.path.join(root, f)
-            os.remove(dst)
-            print(
-                "Removed %s"
-                % (dst if not clean_rel_path.startswith(".") else _relpath(dst))
-            )
+
+    if clean_all and os.path.isdir(libdeps_dir):
+        _clean_dir(libdeps_dir)
+        fs.rmtree(libdeps_dir)
+
     print("Done cleaning")
-    fs.rmtree(clean_dir)
-    env.Exit(0)
 
 
 def AddTarget(  # pylint: disable=too-many-arguments
@@ -65,7 +74,7 @@ def AddTarget(  # pylint: disable=too-many-arguments
     actions,
     title=None,
     description=None,
-    group="Generic",
+    group="General",
     always_build=True,
 ):
     if "__PIO_TARGETS" not in env:
@@ -101,7 +110,13 @@ def DumpTargets(env):
         description="Generate compilation database `compile_commands.json`",
         group="Advanced",
     )
-    targets["clean"] = dict(name="clean", title="Clean", group="Generic")
+    targets["clean"] = dict(name="clean", title="Clean", group="General")
+    targets["cleanall"] = dict(
+        name="cleanall",
+        title="Clean All",
+        group="General",
+        description="Clean a build environment and installed library dependencies",
+    )
     return list(targets.values())
 
 

platformio/builder/tools/piotarget.py

@@ -141,10 +141,11 @@ def configure_command(self, language, src_file):  # pylint: disable=arguments-di
 
         build_flags = self.cxx_flags if language == "c++" else self.cc_flags
 
-        for flag in build_flags:
-            if "-std" in flag:
-                # Standards with GNU extensions are not allowed
-                cmd.append("-" + flag.replace("gnu", "c"))
+        if not self.is_flag_set("--std", flags):
+            # Try to guess the standard version from the build flags
+            for flag in build_flags:
+                if "-std" in flag:
+                    cmd.append("-" + self.convert_language_standard(flag))
 
         cmd.extend(
             ["-D%s" % d for d in self.cpp_defines + self.toolchain_defines[language]]
@@ -224,6 +225,21 @@ def is_check_successful(cmd_result):
         # Cppcheck is configured to return '3' if a defect is found
         return cmd_result["returncode"] in (0, 3)
 
+    @staticmethod
+    def convert_language_standard(flag):
+        cpp_standards_map = {
+            "0x": "11",
+            "1y": "14",
+            "1z": "17",
+            "2a": "20",
+        }
+
+        standard = flag[-2:]
+        # Note: GNU extensions are not supported and converted to regular standards
+        return flag.replace("gnu", "c").replace(
+            standard, cpp_standards_map.get(standard, standard)
+        )
+
     def check(self, on_defect_callback=None):
         self._on_defect_callback = on_defect_callback
 

platformio/commands/check/tools/cppcheck.py

@@ -122,7 +122,7 @@ def cli(  # pylint: disable=too-many-arguments, too-many-branches
             fs.rmtree(build_dir)
 
 
-def _copy_contents(dst_dir, contents):
+def _copy_contents(dst_dir, contents):  # pylint: disable=too-many-branches
     items = {"dirs": set(), "files": set()}
 
     for path in contents:
@@ -134,14 +134,15 @@ def _copy_contents(dst_dir, contents):
     dst_dir_name = os.path.basename(dst_dir)
 
     if dst_dir_name == "src" and len(items["dirs"]) == 1:
-        shutil.copytree(list(items["dirs"]).pop(), dst_dir, symlinks=True)
+        if not os.path.isdir(dst_dir):
+            shutil.copytree(list(items["dirs"]).pop(), dst_dir, symlinks=True)
     else:
         if not os.path.isdir(dst_dir):
             os.makedirs(dst_dir)
         for d in items["dirs"]:
-            shutil.copytree(
-                d, os.path.join(dst_dir, os.path.basename(d)), symlinks=True
-            )
+            src_dst_dir = os.path.join(dst_dir, os.path.basename(d))
+            if not os.path.isdir(src_dst_dir):
+                shutil.copytree(d, src_dst_dir, symlinks=True)
 
     if not items["files"]:
         return

platformio/commands/ci.py

@@ -265,7 +265,8 @@ def _generate_project_main(project_dir, board, framework):
                 fp.write(main_content.strip())
         return project_dir
 
-    async def import_arduino(self, board, use_arduino_libs, arduino_project_dir):
+    @staticmethod
+    async def import_arduino(board, use_arduino_libs, arduino_project_dir):
         board = str(board)
         # don't import PIO Project
         if is_platformio_project(arduino_project_dir):

platformio/commands/home/rpc/handlers/project.py

@@ -336,7 +336,10 @@ def device_monitor(ctx, agents, **kwargs):
     kwargs["baud"] = kwargs["baud"] or 9600
 
     def _tx_target(sock_dir):
-        subcmd_argv = ["remote", "device", "monitor"]
+        subcmd_argv = ["remote"]
+        for agent in agents:
+            subcmd_argv.extend(["--agent", agent])
+        subcmd_argv.extend(["device", "monitor"])
         subcmd_argv.extend(device_helpers.options_to_argv(kwargs, project_options))
         subcmd_argv.extend(["--sock", sock_dir])
         subprocess.call([proc.where_is_program("platformio")] + subcmd_argv)

platformio/commands/remote/command.py

@@ -81,6 +81,7 @@ def _load_tplvars(self):
                     "src_files": self.get_src_files(),
                     "project_src_dir": self.config.get_optional_dir("src"),
                     "project_lib_dir": self.config.get_optional_dir("lib"),
+                    "project_test_dir": self.config.get_optional_dir("test"),
                     "project_libdeps_dir": os.path.join(
                         self.config.get_optional_dir("libdeps"), self.env_name
                     ),

platformio/ide/projectgenerator.py

@@ -115,7 +115,7 @@ endif()
 % end
 
 FILE(GLOB_RECURSE SRC_LIST
-%   for path in (project_src_dir, project_lib_dir):
+%   for path in (project_src_dir, project_lib_dir, project_test_dir):
     {{  _normalize_path(path) + "/*.*" }}
 %   end
 )