Merge pull request #1808 from buildtesters/specify_file_to_add_to_buildspec_cache

Add support for specifying a file path via -f/--file when loading buildspec into cache

Author: shahzebsiddiqui

bash_completion.sh

@@ -413,8 +413,8 @@ _buildtest ()
            COMPREPLY=( $( compgen -W "${opts}" -- "${cur}" ) );;
          # completion for rest of arguments
          *)
-           local longopts="--buildspec --count --directory --executors --filter --filterfields --format --formatfields --group-by-executor --group-by-tags --help --helpfilter --helpformat --no-header --pager --paths --quiet --rebuild --row-count --tags --terse"
-           local shortopts="-b -d -e -h -n -p -q -r -t"
+           local longopts="--buildspec --count --directory --executors --file --filter --filterfields --format --formatfields --group-by-executor --group-by-tags --help --helpfilter --helpformat --no-header --pager --paths --quiet --rebuild --row-count --tags --terse"
+           local shortopts="-b -d -e -f -h -n -p -q -r -t"
            local cmds="invalid"
 
            COMPREPLY=( $( compgen -W "${cmds} ${longopts} ${shortopts}" -- "${cur}" ) )

buildtest/cli/__init__.py

@@ -1294,6 +1294,14 @@ def buildspec_find_menu(self, buildspec_find_parser):
                         "help": "Specify root buildspecs (directory) path to load buildspecs into buildspec cache.",
                     },
                 ),
+                (
+                    ["-f", "--file"],
+                    {
+                        "type": str,
+                        "action": "append",
+                        "help": "Specify buildspec file to load into buildspec cache.",
+                    },
+                ),
             ],
         }
 

buildtest/cli/buildspec.py

@@ -50,6 +50,7 @@ def __init__(
         filterfields=None,
         formatfields=None,
         directory=None,
+        buildspec_files=None,
         header=None,
         terse=None,
         pager=None,
@@ -71,6 +72,7 @@ def __init__(
             filterfields (str, optional): The filter options specified via ``buildtest buildspec find --filter`` that contains list of key value pairs for filtering buildspecs
             formatfields (str, optional): The format options used for formating table. The format option is a comma separated list of format fields specified via ``buildtest buildspec find --format``
             directory (list, optional): List of directories to search for buildspecs. This argument contains value of ``buildtest buildspec find --directory``
+            buildspec_files (list, optional): List of buildspec files to add to cache. This argument contains value of ``buildtest buildspec find --file``
             headers (bool, optional):  Option to control whether header are printed in terse output. This argument contains value of ``buildtest buildspec find --no-header``
             terse (bool, optional): Enable terse mode when printing output. In this mode we don't print output in table format instead output is printed in parseable format. This option can be specified via ``buildtest buildspec find --terse``
             color (str, optional): An instance of a string class that selects the color to use when printing table output
@@ -107,6 +109,8 @@ def __init__(
         # list of buildspec directories to search for .yml files
         self.paths = []
 
+        self.buildspec_files = buildspec_files
+
         # stores invalid buildspecs and the error messages
         self.invalid_buildspecs = {}
 
@@ -149,12 +153,12 @@ def load_paths(self):
         if not self.directory:
             self.paths += BUILDSPEC_DEFAULT_PATH
 
-        # for every root buildspec defined in configuration or via --root option,
+        # for every root buildspec defined in configuration or via --directory option,
         # we resolve path and if path exist add to self.paths. The path must be a
         # directory. If its file, we ignore it
         if self.directory:
-            for root in self.directory:
-                path = resolve_path(root, exist=False)
+            for dirname in self.directory:
+                path = resolve_path(dirname, exist=False)
                 if not os.path.exists(path):
                     console.print(f"[red]Path: {path} does not exist!")
 
@@ -169,6 +173,31 @@ def build(self):
         rebuild cache we remove the file and recreate cache. If cache file
         exists, we simply load from cache
         """
+        buildspecs = []
+        # this method will check if buildspec_files are valid files and end with .yml.
+        # If it's not a file or does not end with .yml we skip the file and report a message
+
+        if self.buildspec_files:
+            for buildspec in self.buildspec_files:
+                path = resolve_path(buildspec, exist=False)
+                if not os.path.exists(path):
+                    console.print(f"[red]Path: {path} does not exist!")
+                    continue
+                if not is_file(path):
+                    console.print(
+                        f"[red]{path} is not a file, please specify a file when adding buildspec to cache"
+                    )
+                    continue
+                if not path.endswith(".yml"):
+                    console.print(
+                        f"[red]{path} does not end in .yml extension, please specify a valid buildspec file"
+                    )
+                    continue
+
+                buildspecs.append(path)
+
+        # set self.buildspec_files to list of valid buildspec files which will be used to build cache
+        self.buildspec_files = buildspecs
 
         # implements buildtest buildspec find --rebuild which removes cache file
         # before finding all buildspecs. We only remove file if file exists
@@ -192,7 +221,7 @@ def build(self):
 
     def _discover_buildspecs(self):
         """This method retrieves buildspecs based on ``self.paths`` which is a
-        list of directory paths to search. If ``--root`` is specified
+        list of directory paths to search. If ``--directory`` is specified
         we process each argument and recursively find all .yml files
         """
 
@@ -210,6 +239,9 @@ def _discover_buildspecs(self):
                 buildspec = walk_tree(path, ".yml")
                 buildspecs += buildspec
 
+        if self.buildspec_files:
+            buildspecs += self.buildspec_files
+
         if not self.terse:
             print(f"Buildspec Paths: {self.paths}")
 
@@ -1405,6 +1437,7 @@ def buildspec_find(args, configuration):
         filterfields=args.filter,
         formatfields=args.format,
         directory=args.directory,
+        buildspec_files=args.file,
         configuration=configuration,
         header=args.no_header,
         terse=args.terse,

buildtest/main.py

@@ -252,10 +252,17 @@ def setup(args):
     # build buildspec cache file automatically if it doesn't exist
     if not is_file(BUILDSPEC_CACHE_FILE):
         root_buildspecs = []
+        buildspec_files = []
         if hasattr(args, "directory"):
             root_buildspecs = args.directory
+        if hasattr(args, "file"):
+            buildspec_files = args.file
 
-        BuildspecCache(directory=root_buildspecs, configuration=configuration)
+        BuildspecCache(
+            directory=root_buildspecs,
+            buildspec_files=buildspec_files,
+            configuration=configuration,
+        )
 
     return system, configuration, buildtest_editor, report_file
 

docs/configuring_buildtest/overview.rst

@@ -220,10 +220,12 @@ If you want buildtest to always rebuild cache you can set the following in your
 The configuration options such as ``count``, ``format``, ``terse`` can  be tweaked to your preference. These configuration values
 can be overridden by command line option.
 
-.. _buildspec_roots:
+.. _search_buildspecs_when_building_cache:
 
-Specify Root Directories for searching buildspecs
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Searching for buildspecs when building Buildspec Cache
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+When building the buildspec cache, buildtest will search for buildspecs in a list of directories specified in the configuration file.
 
 Buildtest will search for buildspecs by recursively searching for files with **.yml** extension. The ``directory`` property in configuration file
 is a list of directories to search for buildspecs. The ``directory`` property is not **required** in configuration file, but it can be a good

docs/gettingstarted/buildspecs_interface.rst

@@ -78,8 +78,8 @@ Shown below is an example output.
     .. command-output:: buildtest buildspec find --buildspec
        :ellipsis: 11
 
-Find root paths where buildspecs are searched
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Adding buildspecs to cache
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 The ``buildtest buildspec find --paths`` will display a list of root directories buildtest will search for
 buildspecs when running ``buildtest buildspec find``. One can define these directories in the configuration file
@@ -89,10 +89,8 @@ or pass them via command line.
 
     .. command-output:: buildtest buildspec find --paths
 
-buildtest will search buildspecs in :ref:`buildspecs root <buildspec_roots>` defined in your configuration,
-which is a list of directory paths to search for buildspecs.
-If you want to load buildspecs from a directory path, you can specify a directory
-via ``--directory`` option in the format: ``buildtest buildspec find --directory <path>``.
+buildtest will :ref:`search buildspecs when building cache <search_buildspecs_when_building_cache>` that can be configured via
+configuration or command line. If you want to load buildspecs from a directory, you can use the ``--directory`` option.
 buildtest will rebuild cache when `--directory` option is specified. Note that to rebuild cache you typically
 need to pass **--rebuild** option but that is not required when using **--directory** option because we want
 buildtest to load buildspecs into cache.
@@ -112,6 +110,23 @@ Let's rebuild the cache again by running ``buildtest buildspec find`` which will
 
 .. command-output:: buildtest buildspec find --rebuild --quiet
 
+In addition to ``--directory`` option, one can specify a list of files to load into cache using the ``--file`` option. This can be useful
+if you want to load specific buildspecs into cache without having to specify ``--directory``. You can use ``--file`` option with ``--directory``
+and buildtest will recursively search directories and load files specified in ``--file`` option.
+
+If you specify an invalid file path, a directory or file without ``.yml`` extension, buildtest will report a message and skip to next file.
+Shown below, we specify a list of files to load into cache using ``--file`` option.
+
+.. dropdown:: ``buildtest buildspec find --file $BUILDTEST_ROOT/tutorials/vars.yml``
+
+    .. command-output:: buildtest buildspec find --file $BUILDTEST_ROOT/tutorials/vars.yml
+
+    We can confirm the file is loaded into cache using the `-b` option which list all buildspecs in cache and pipe via `grep` to search for `vars.yml`. Note that
+    we specify ``--count=-1`` to show all buildspecs in cache.
+
+    .. command-output:: buildtest buildspec find -b --terse --count=-1 | grep vars.yml
+       :shell:
+
 Filtering buildspec
 ~~~~~~~~~~~~~~~~~~~~
 

tests/cli/test_buildspec.py

@@ -348,18 +348,28 @@ def test_edit_file():
 
 
 @pytest.mark.cli
-def test_buildspec_find_roots():
+def test_buildspec_find_by_directory_and_files():
     root_buildspecs = [
         os.path.join(BUILDTEST_ROOT, "tests", "buildsystem"),
         os.path.join(BUILDTEST_ROOT, "tutorials"),
     ]
-    # buildtest buildspec find --root $BUILDTEST_ROOT/tests/buildsystem --root $BUILDTEST_ROOT/tutorials
+    # list of buildspec files to add to cache, we have one valid file that exists, one with invalid extension and one file that doesn't exist
+    bp_files = [
+        os.path.join(BUILDTEST_ROOT, "tutorials", "vars.yml"),
+        os.path.join(BUILDTEST_ROOT, "README.rst"),  # invalid extension
+        os.path.join(BUILDTEST_ROOT, "badfile.yml"),  # file doesn't exist
+        os.path.join(BUILDTEST_ROOT),  # directory path
+    ]
+    # buildtest buildspec find --directory $BUILDTEST_ROOT/tests/buildsystem --directory $BUILDTEST_ROOT/tutorials
     BuildspecCache(
-        directory=root_buildspecs, configuration=configuration, rebuild=False
+        directory=root_buildspecs,
+        buildspec_files=bp_files,
+        configuration=configuration,
+        rebuild=False,
     )
 
     with pytest.raises(BuildTestError):
-        # buildtest buildspec find --root $BUILDTEST_ROOT/README.rst --root $BUILDTEST_ROOT/environment.yml
+        # buildtest buildspec find --directory $BUILDTEST_ROOT/README.rst --directory $BUILDTEST_ROOT/environment.yml
         BuildspecCache(
             directory=[
                 os.path.join(BUILDTEST_ROOT, "README.rst"),