app: manifest: Add the ability to display "untracked" files and folders

This new command-line switch implements something similar to `git
status`'s untracked files display.

The code examines the workspace and prints out all files and folders
that are not inside of a west project.

Signed-off-by: Carles Cufi <[email protected]>

Author: carlescufi

src/west/app/project.py

@@ -584,6 +584,13 @@ def __init__(self):
               If this file uses imports, it will not contain all the
               manifest data.
 
+            - --untracked print all files and folders inside the workspace that
+              are not tracked or managed by west. This effectively means any
+              file or folder that lays outside all of the projects' folders on
+              disk. This is similar to `git status` for untracked files. The
+              output format is relative to the current working directory and is
+              stable and suitable as input for scripting.
+
             If the manifest file does not use imports, and all project
             revisions are SHAs, the --freeze and --resolve output will
             be identical after a "west update".
@@ -604,6 +611,9 @@ def do_add_parser(self, parser_adder):
                            exiting with an error if there are issues''')
         group.add_argument('--path', action='store_true',
                            help="print the top level manifest file's path")
+        group.add_argument('--untracked', action='store_true',
+                           help='''print any files and folders not managed or
+                           tracked by west''')
 
         group = parser.add_argument_group('options for --resolve and --freeze')
         group.add_argument('-o', '--out',
@@ -624,6 +634,8 @@ def do_run(self, args, user_args):
         elif args.freeze:
             self._die_if_manifest_project_filter('freeze')
             self._dump(args, manifest.as_frozen_yaml(**dump_kwargs))
+        elif args.untracked:
+            self._untracked()
         elif args.path:
             self.inf(manifest.path)
         else:
@@ -640,6 +652,70 @@ def _die_if_manifest_project_filter(self, action):
                      'the manifest while projects are made inactive by the '
                      'project filter.')
 
+    def _untracked(self):
+        ''' "Performs a top-down search of the west topdir,
+        ignoring every folder that corresponds to a west project.
+        '''
+        ppaths = []
+        untracked = []
+        for project in self._projects(None):
+            # We do not check for self.manifest.is_active(project) because
+            # inactive projects are still considered "tracked folders".
+            ppaths.append(Path(project.abspath))
+
+        # Remove projects whose folders are not present on disk. If the project
+        # is not present on disk then it has no relevance when looking for
+        # untracked files and folders, since for those to exist the path must
+        # exist as well.
+        ppaths = [p for p in ppaths if p.exists()]
+        # Since west tolerates nested projects (i.e. a project inside the folder
+        # of another project) we must sort the project paths to ensure that we
+        # hit the "enclosing" project first when iterating.
+        ppaths.sort()
+
+        def _find_untracked(folder):
+            '''There are three cases for each element in a folder:
+            - It's a project -> Do nothing, ignore the folder.
+            - There are no projects inside -> add to untracked list.
+            - It's not a project folder but there are some projects inside it -> recurse.
+            The folder argument cannot be inside a project, otherwise all bets are off.
+            '''
+            self.dbg(f'looking for untracked files/folders in: {folder}')
+            for e in [e.absolute() for e in folder.iterdir()]:
+                if not e.is_dir() or e.is_symlink():
+                    untracked.append(e)
+                    continue
+                self.dbg(f'processing folder: {e}')
+                for ppath in ppaths:
+                    if e.samefile(ppath):
+                        # We hit a project root folder, skip it.
+                        break
+                    elif e in ppath.parents:
+                        self.dbg(f'recursing into: {e}')
+                        _find_untracked(e)
+                        break
+                else:
+                    # This is not a project and there is no project inside.
+                    # Add to untracked elements.
+                    untracked.append(e)
+                    continue
+
+        # Avoid using Path.walk() since that returns all files and folders under
+        # a particular folder, which is overkill in our case. Instead, recurse
+        # only when required.
+        _find_untracked(Path(self.topdir))
+
+        # Remove the .west folder, which is maintained by west
+        try:
+            untracked.remove((Path(self.topdir) / Path(WEST_DIR)).resolve())
+        except ValueError:
+            self.die(f'Folder {WEST_DIR} not found in workspace')
+
+        # Sort the results for displaying to the user.
+        untracked.sort()
+        for u in untracked:
+            self.inf(u.relative_to(Path.cwd(), walk_up=True))
+
     def _dump(self, args, to_dump):
         if args.out:
             with open(args.out, 'w') as f:
@@ -881,7 +957,10 @@ def __init__(self):
             'status',
             '"git status" for one or more projects',
             '''Runs "git status" for each of the specified projects.
-            Unknown arguments are passed to "git status".''',
+            Unknown arguments are passed to "git status".
+
+            Note: If you are looking to find untracked files and folders
+            in the workspace use "west manifest --untracked".''',
             accepts_unknown_args=True,
         )