Add support for git sparse checkout

Added a new, optional external property, sparse. sparse points to a file (relative to external repository local directory) with git sparse checkout patterns (e.g., directory names to include).
If the sparse keyword is included, enable sparse checkout for that repository and use git read-tree to perform the sparse checkout.

Added a new test for sparse checkout. The test is a repo with two externals which are both tag2 of the simple_ext repo with one of them configured for a sparse checkout. The test makes sure that both externals have the correct files.

Updated the documentation for the externals configuration file.

User interface changes?: Yes
New sparse keyword is optional with a blank default value which preserves current functionality.

Fixes: #120

Testing:
test removed: None
unit tests: All passed
system tests: All passed, including new test for sparse feature
manual testing: Several manual tests of NASA test cases.

Author: goldy

README.md

@@ -85,7 +85,7 @@ The root of the source tree will be referred to as `${SRC_ROOT}` below.
     description file:
 
         $ cd ${SRC_ROOT}
-        $ ./manage_externals/checkout_externals --excernals my-externals.cfg
+        $ ./manage_externals/checkout_externals --externals my-externals.cfg
 
   * Status summary of the repositories managed by checkout_externals:
 
@@ -202,6 +202,21 @@ The root of the source tree will be referred to as `${SRC_ROOT}` below.
     Then the main 'externals' field in the top level repo should point to
     'sub-externals.cfg'.
 
+  * from_submodule (True / False) : used to pull the repo_url, local_path,
+    and hash properties for this external from the .gitmodules file in
+    this repository. Note that the section name (the entry in square
+    brackets) must match the name in the .gitmodules file.
+    If from_submodule is True, the protocol must be git and no repo_url,
+    local_path, hash, branch, or tag entries are allowed.
+    Default: False
+
+  * sparse (string) : used to control a sparse checkout. This optional
+    entry should point to a filename (path relative to local_path) that
+    contains instructions on which repository paths to include (or
+    exclude) from the working tree.
+    See the "SPARSE CHECKOUT" section of https://git-scm.com/docs/git-read-tree
+    Default: sparse checkout is disabled
+
   * Lines begining with '#' or ';' are comments and will be ignored.
 
 # Obtaining this tool, reporting issues, etc.

manic/checkout.py

@@ -227,6 +227,21 @@ def commandline_arguments(args=None):
     Now, %(prog)s will process Externals.cfg and also process
     Externals_LIBX.cfg as if it was a sub-external.
 
+  * from_submodule (True / False) : used to pull the repo_url, local_path,
+    and hash properties for this external from the .gitmodules file in
+    this repository. Note that the section name (the entry in square
+    brackets) must match the name in the .gitmodules file.
+    If from_submodule is True, the protocol must be git and no repo_url,
+    local_path, hash, branch, or tag entries are allowed.
+    Default: False
+
+  * sparse (string) : used to control a sparse checkout. This optional
+    entry should point to a filename (path relative to local_path) that
+    contains instructions on which repository paths to include (or
+    exclude) from the working tree.
+    See the "SPARSE CHECKOUT" section of https://git-scm.com/docs/git-read-tree
+    Default: sparse checkout is disabled
+
   * Lines beginning with '#' or ';' are comments and will be ignored.
 
 # Obtaining this tool, reporting issues, etc.

manic/externals_description.py

@@ -351,6 +351,7 @@ class ExternalsDescription(dict):
     REPO_URL = 'repo_url'
     REQUIRED = 'required'
     TAG = 'tag'
+    SPARSE = 'sparse'
 
     PROTOCOL_EXTERNALS_ONLY = 'externals_only'
     PROTOCOL_GIT = 'git'
@@ -374,6 +375,7 @@ class ExternalsDescription(dict):
                              TAG: 'string',
                              BRANCH: 'string',
                              HASH: 'string',
+                             SPARSE: 'string',
                             }
                      }
 
@@ -562,6 +564,8 @@ def _check_optional(self):
                 self[field][self.REPO][self.HASH] = EMPTY_STR
             if self.REPO_URL not in self[field][self.REPO]:
                 self[field][self.REPO][self.REPO_URL] = EMPTY_STR
+            if self.SPARSE not in self[field][self.REPO]:
+                self[field][self.REPO][self.SPARSE] = EMPTY_STR
 
             # from_submodule has a complex relationship with other fields
             if self.SUBMODULE in self[field]:

manic/repository.py

@@ -21,6 +21,7 @@ def __init__(self, component_name, repo):
         self._branch = repo[ExternalsDescription.BRANCH]
         self._hash = repo[ExternalsDescription.HASH]
         self._url = repo[ExternalsDescription.REPO_URL]
+        self._sparse = repo[ExternalsDescription.SPARSE]
 
         if self._url is EMPTY_STR:
             fatal_error('repo must have a URL')

manic/repository_git.py

@@ -316,8 +316,11 @@ def _checkout_ref(self, repo_dir, verbosity, submodules):
         else:
             self._checkout_external_ref(verbosity, submodules)
 
+        if self._sparse:
+            self._sparse_checkout(repo_dir, verbosity)
         os.chdir(cwd)
 
+
     def _checkout_local_ref(self, verbosity, submodules):
         """Checkout the reference considering the local repo only. Do not
         fetch any additional remotes or specify the remote when
@@ -362,6 +365,20 @@ def _checkout_external_ref(self, verbosity, submodules):
             ref = '{0}/{1}'.format(remote_name, ref)
         self._git_checkout_ref(ref, verbosity, submodules)
 
+    def _sparse_checkout(self, repo_dir, verbosity):
+        """Use git read-tree to thin the working tree."""
+        cwd = os.getcwd()
+
+        cmd = ['cp', self._sparse, os.path.join(repo_dir,
+                                                '.git/info/sparse-checkout')]
+        if verbosity >= VERBOSITY_VERBOSE:
+            printlog('    {0}'.format(' '.join(cmd)))
+        execute_subprocess(cmd)
+        os.chdir(repo_dir)
+        self._git_sparse_checkout(verbosity)
+
+        os.chdir(cwd)
+
     def _check_for_valid_ref(self, ref, remote_name=None):
         """Try some basic sanity checks on the user supplied reference so we
         can provide a more useful error message than calledprocess
@@ -776,6 +793,18 @@ def _git_checkout_ref(ref, verbosity, submodules):
         if submodules:
             GitRepository._git_update_submodules(verbosity)
 
+    @staticmethod
+    def _git_sparse_checkout(verbosity):
+        """Configure repo via read-tree."""
+        cmd = ['git', 'config', 'core.sparsecheckout', 'true']
+        if verbosity >= VERBOSITY_VERBOSE:
+            printlog('    {0}'.format(' '.join(cmd)))
+        execute_subprocess(cmd)
+        cmd = ['git', 'read-tree', '-mu', 'HEAD']
+        if verbosity >= VERBOSITY_VERBOSE:
+            printlog('    {0}'.format(' '.join(cmd)))
+        execute_subprocess(cmd)
+
     @staticmethod
     def _git_update_submodules(verbosity):
         """Run git submodule update for the side effect of updating this

manic/sourcetree.py

@@ -45,6 +45,7 @@ def __init__(self, root_dir, name, ext_description, svn_ignore_ancestry):
         self._externals = EMPTY_STR
         self._externals_sourcetree = None
         self._stat = ExternalStatus()
+        self._sparse = None
         # Parse the sub-elements
 
         # _path : local path relative to the containing source tree