app: manifest: Add the ability to display "untracked" files and dirs

This new command-line switch implements something similar to `git
status`'s untracked files display.

The code examines the workspace and prints out all files and directories
that are not inside of a west project.

Signed-off-by: Carles Cufi <[email protected]>

Author: carlescufi

src/west/app/project.py

@@ -584,6 +584,14 @@ def __init__(self):
               If this file uses imports, it will not contain all the
               manifest data.
 
+            - --untracked: print all files and directories inside the workspace that
+              are not tracked or managed by west. This effectively means any
+              file or directory that is outside all of the projects' directories on
+              disk (regardless of whether those projects are active or inactive).
+              This is similar to `git status` for untracked files. The
+              output format is relative to the current working directory and is
+              stable and suitable as input for scripting.
+
             If the manifest file does not use imports, and all project
             revisions are SHAs, the --freeze and --resolve output will
             be identical after a "west update".
@@ -604,6 +612,9 @@ def do_add_parser(self, parser_adder):
                            exiting with an error if there are issues''')
         group.add_argument('--path', action='store_true',
                            help="print the top level manifest file's path")
+        group.add_argument('--untracked', action='store_true',
+                           help='''print all files and directories not managed or
+                           tracked by west''')
 
         group = parser.add_argument_group('options for --resolve and --freeze')
         group.add_argument('-o', '--out',
@@ -624,6 +635,8 @@ def do_run(self, args, user_args):
         elif args.freeze:
             self._die_if_manifest_project_filter('freeze')
             self._dump(args, manifest.as_frozen_yaml(**dump_kwargs))
+        elif args.untracked:
+            self._untracked()
         elif args.path:
             self.inf(manifest.path)
         else:
@@ -640,6 +653,70 @@ def _die_if_manifest_project_filter(self, action):
                      'the manifest while projects are made inactive by the '
                      'project filter.')
 
+    def _untracked(self):
+        ''' "Performs a top-down search of the west topdir,
+        ignoring every directory that corresponds to a west project.
+        '''
+        ppaths = []
+        untracked = []
+        for project in self._projects(None):
+            # We do not check for self.manifest.is_active(project) because
+            # inactive projects are still considered "tracked directories".
+            ppaths.append(Path(project.abspath))
+
+        # Since west tolerates nested projects (i.e. a project inside the directory
+        # of another project) we must sort the project paths to ensure that we
+        # hit the "enclosing" project first when iterating.
+        ppaths.sort()
+
+        def _find_untracked(directory):
+            '''There are three cases for each element in a directory:
+            - It's a project -> Do nothing, ignore the directory.
+            - There are no projects inside -> add to untracked list.
+            - It's not a project directory but there are some projects inside it -> recurse.
+            The directory argument cannot be inside a project, otherwise all bets are off.
+            '''
+            self.dbg(f'looking for untracked files/directories in: {directory}')
+            for e in [e.absolute() for e in directory.iterdir()]:
+                if not e.is_dir() or e.is_symlink():
+                    untracked.append(e)
+                    continue
+                self.dbg(f'processing directory: {e}')
+                for ppath in ppaths:
+                    # We cannot use samefile() because it requires the file
+                    # to exist (not always the case with inactive or even
+                    # uncloned projects).
+                    if ppath == e:
+                        # We hit a project root directory, skip it.
+                        break
+                    elif e in ppath.parents:
+                        self.dbg(f'recursing into: {e}')
+                        _find_untracked(e)
+                        break
+                else:
+                    # This is not a project and there is no project inside.
+                    # Add to untracked elements.
+                    untracked.append(e)
+                    continue
+
+        # Avoid using Path.walk() since that returns all files and directories under
+        # a particular directory, which is overkill in our case. Instead, recurse
+        # only when required.
+        _find_untracked(Path(self.topdir))
+
+        # Exclude the .west directory, which is maintained by west
+        try:
+            untracked.remove((Path(self.topdir) / Path(WEST_DIR)).resolve())
+        except ValueError:
+            self.die(f'Directory {WEST_DIR} not found in workspace')
+
+        # Sort the results for displaying to the user.
+        untracked.sort()
+        for u in untracked:
+            # We cannot use Path.relative_to(p, walk_up=True) because the
+            # walk_up parameter was only added in 3.12
+            self.inf(os.path.relpath(u, Path.cwd()))
+
     def _dump(self, args, to_dump):
         if args.out:
             with open(args.out, 'w') as f:
@@ -881,7 +958,10 @@ def __init__(self):
             'status',
             '"git status" for one or more projects',
             '''Runs "git status" for each of the specified projects.
-            Unknown arguments are passed to "git status".''',
+            Unknown arguments are passed to "git status".
+
+            Note: If you are looking to find untracked files and directories
+            in the workspace use "west manifest --untracked".''',
             accepts_unknown_args=True,
         )
 
@@ -1747,7 +1827,8 @@ def do_run(self, args, user_args):
 
 To do the same with:
 
-- git grep --untracked: west grep --untracked foo
+- git grep --untracked: west grep --untracked foo (see also west manifest --untracked
+  for finding untracked files outside the project directories)
 - ripgrep:              west grep --tool ripgrep foo
 - grep --recursive:     west grep --tool grep foo
 

tests/test_project.py

@@ -5,6 +5,7 @@
 import re
 import shutil
 import subprocess
+import sys
 import textwrap
 from pathlib import Path, PurePath
 
@@ -239,6 +240,122 @@ def test_list_sha(west_update_tmpdir):
     assert cmd('list -f {sha}').startswith("N/A")
 
 
+def test_manifest_untracked(west_update_tmpdir):
+
+    def check(expected, cwd=None):
+        out_lines = cmd("manifest --untracked", cwd=cwd).splitlines()
+        assert out_lines == expected
+
+    # Ensure that all projects are active
+    projs = cmd('list -f {name}').splitlines()
+    assert projs == ['manifest', 'Kconfiglib', 'tagged_repo', 'net-tools']
+
+    # Disable Kconfiglib
+    cmd('config manifest.group-filter -- -Kconfiglib-group')
+
+    # Ensure that Kconfiglib is inactive
+    projs = cmd('list -f {name}').splitlines()
+    assert projs == ['manifest', 'tagged_repo', 'net-tools']
+    projs = cmd('list --all -f {name}').splitlines()
+    assert projs == ['manifest', 'Kconfiglib', 'tagged_repo', 'net-tools']
+
+    topdir = Path(west_update_tmpdir)
+
+    # No untracked files yet
+    check(list())
+
+    (topdir / "dir").mkdir()
+    # Untracked dir
+    check(['dir'])
+
+    (topdir / "unt").mkdir()
+    (topdir / "unt" / "file.yml").touch()
+    # Ensure we show the dir as untracked, not the file
+    check(['dir', 'unt'])
+
+    # Add a file to see it's displayed correctly as untracked
+    (topdir / "file.txt").touch()
+    check(['dir', 'file.txt', 'unt'])
+    (topdir / 'subdir' / "z.py").touch()
+    check(['dir', 'file.txt', str(Path('subdir/z.py')), 'unt'])
+
+    (topdir / "subdir" / "new").mkdir()
+    (topdir / "subdir" / "new" / "afile.txt").touch()
+    check(['dir', 'file.txt', str(Path('subdir/new')),
+           str(Path('subdir/z.py')), 'unt'])
+
+    # Check relative paths
+    check([str(Path('../dir')), str(Path('../file.txt')),
+           str(Path('../subdir/new')), str(Path('../subdir/z.py')), '.'],
+          cwd=str(Path("unt/")))
+
+    # Add a file to an existing project, ignored by --untracked
+    (topdir / "net-tools" / "test_manifest_untracked.file").touch()
+    check(['dir', 'file.txt', str(Path('subdir/new')), str(Path('subdir/z.py')),
+           'unt'])
+
+    kconfiglib = Path(topdir / "subdir" / "Kconfiglib")
+    # Same but with an inactive project
+    (kconfiglib / "test_manifest_untracked.file").touch()
+    check(['dir', 'file.txt', str(Path('subdir/new')), str(Path('subdir/z.py')),
+           'unt'])
+
+    # Copy (via clone) a full Git repo so we verify that those are also
+    # displayed as untracked.
+    clone(topdir / "net-tools", Path('subdir/acopy'))
+    clone(topdir / "net-tools", Path('tmpcopy'))
+
+    check(['dir', 'file.txt', str(Path('subdir/acopy')), str(Path('subdir/new')),
+           str(Path('subdir/z.py')), 'tmpcopy', 'unt'])
+
+    # Empty a project so it's not a Git repo anymore
+    (topdir / "net-tools" / ".git").rename(topdir / "net-tools" / "former-git")
+    # Should make no difference
+    check(['dir', 'file.txt', str(Path('subdir/acopy')), str(Path('subdir/new')),
+           str(Path('subdir/z.py')), 'tmpcopy', 'unt'])
+
+    # Same with an inactive project
+    (kconfiglib / ".git").rename(kconfiglib / "former-git")
+    # Should make no difference
+    check(['dir', 'file.txt', str(Path('subdir/acopy')), str(Path('subdir/new')),
+           str(Path('subdir/z.py')), 'tmpcopy', 'unt'])
+
+    # Even if we make the whole inactive project disappear it should make no
+    # difference at all, except that the renamed dir will show up.
+    (kconfiglib).rename(topdir / "subdir" / "other")
+    check(['dir', 'file.txt', str(Path('subdir/acopy')), str(Path('subdir/new')),
+           str(Path('subdir/other')), str(Path('subdir/z.py')), 'tmpcopy', 'unt'])
+
+@pytest.mark.skipif(sys.platform.startswith("win"), reason="symbolic links not tested on Windows")
+def test_manifest_untracked_with_symlinks(west_update_tmpdir):
+
+    def check(expected, cwd=None):
+        out_lines = cmd("manifest --untracked", cwd=cwd).splitlines()
+        assert out_lines == expected
+
+    # Disable Kconfiglib to have an inactive project
+    cmd('config manifest.group-filter -- -Kconfiglib-group')
+
+    # Ensure that Kconfiglib is inactive
+    projs = cmd('list -f {name}').splitlines()
+    assert projs == ['manifest', 'tagged_repo', 'net-tools']
+
+    # Create a folder symlink
+    Path('asl').symlink_to(Path('subdir/Kconfiglib'))
+    # Create another one
+    Path('anothersl').symlink_to(Path('../'))
+    # Yet another one
+    Path('subdir/yetanothersl').symlink_to(Path('tagged_repo'))
+    # check that symlinks are displayed like any other regular untracked file or
+    # directory
+    check(['anothersl', 'asl', str(Path('subdir/yetanothersl'))])
+
+    # File symlink tests, should all be displayed as untracked as well
+    Path('filesl.yml').symlink_to(Path('zephyr/west.yml'))
+    Path('subdir/afsl.py').symlink_to(Path('net-tools/scripts/test.py'))
+    check(['anothersl', 'asl', 'filesl.yml', str(Path('subdir/afsl.py')),
+           str(Path('subdir/yetanothersl'))])
+
 def test_manifest_freeze(west_update_tmpdir):
     # We should be able to freeze manifests.
     actual = cmd('manifest --freeze').splitlines()