asyncio.gather

coverage

coverage

pep8

Author: scivision

.travis.yml

@@ -6,13 +6,14 @@ git:
   depth: 25
   quiet: true
 
+python: 3.7
+
 matrix:
   include:
   - os: linux
     python: 3.6
   - os: linux
     name: Integration install
-    python: 3.7
     install:
     - python setup.py install
     - pip install gitutils[tests]
@@ -21,8 +22,7 @@ matrix:
     - python -m pytest $TRAVIS_BUILD_DIR/tests -r a -v
   - os: linux
     name: PEP8 MyPy Coverage
-    python: 3.7
-    install: pip install -e .[tests,cov]
+    install: pip install -e .[tests,cov,github]
     script:
     - flake8
     - mypy .

RemoveCollaborators.py

@@ -53,7 +53,7 @@ def main():
         if modify:
             if repo.archived:
                 logging.error(f'could not remove collabs from archived {repo.full_name}')
-                webbrowser.open_new_tab('https://github.com/'+repo.full_name+'/settings')
+                webbrowser.open_new_tab('https://github.com/' + repo.full_name + '/settings')
                 continue
 
             for admin in admins:

branch.py

@@ -3,40 +3,25 @@
 report on git repos not on the expected branch e.g. 'master'
 """
 from argparse import ArgumentParser
-from gitutils import findbranch
-from pathlib import Path
-import asyncio
-import os
-import sys
-
-
-async def find_branch(branch: str, path: Path):
-    path = Path(path).expanduser()
-
-    async for b in findbranch(branch, path):
-        print(b[0], ' => ', b[1])
+import logging
+from gitutils.branch import coro_local
+from gitutils.runner import runner
 
 
 def main():
     p = ArgumentParser()
     p.add_argument('codepath', help='path to code root', nargs='?', default='~/code')
     p.add_argument('mainbranch', nargs='?',
                    default='master', help='name of your main branch')
+    p.add_argument('-v', '--verbose', action='store_true')
     P = p.parse_args()
 
-    if os.name == 'nt' and (3, 7) <= sys.version_info < (3, 8):
-        asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy())
+    if P.verbose:
+        logging.basicConfig(level=logging.DEBUG)
 
-    if sys.version_info >= (3, 7):
-        asyncio.run(find_branch(P.mainbranch, P.codepath))
-    else:
-        if os.name == 'nt':
-            loop = asyncio.ProactorEventLoop()
-        else:
-            loop = asyncio.new_event_loop()
-            asyncio.get_child_watcher().attach_loop(loop)
-        loop.run_until_complete(find_branch(P.mainbranch, P.codepath))
-        loop.close()
+    branches = runner(coro_local, P.mainbranch, P.codepath)
+    for b in branches:
+        print(b[0], ' => ', b[1])
 
 
 if __name__ == '__main__':

check.py

@@ -3,13 +3,11 @@
 for a root directory, assumes all subdirectories are Git repos
 and "git fetch --dry-run" each
 """
-import sys
-import os
 import logging
-import asyncio
 from argparse import ArgumentParser
 
-from gitutils.pull import find_remote
+from gitutils.pull import coro_remote
+from gitutils.runner import runner
 
 MODE = ['fetch', '--dry-run']
 
@@ -23,19 +21,8 @@ def main():
     if P.verbose:
         logging.basicConfig(level=logging.DEBUG)
 
-    if os.name == 'nt' and (3, 7) <= sys.version_info < (3, 8):
-        asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy())
-
-    if sys.version_info >= (3, 7):
-        asyncio.run(find_remote(MODE, P.codepath))
-    else:
-        if os.name == 'nt':
-            loop = asyncio.ProactorEventLoop()
-        else:
-            loop = asyncio.new_event_loop()
-            asyncio.get_child_watcher().attach_loop(loop)
-        loop.run_until_complete(find_remote(MODE, P.codepath))
-        loop.close()
+    remotes = runner(coro_remote, MODE, P.codepath)
+    print('\n'.join(map(str, remotes)))
 
 
 if __name__ == '__main__':

fetch.py

@@ -3,13 +3,11 @@
 for a root directory, assumes all subdirectories are Git repos
 and "git fetch" each of them.
 """
-import sys
-import os
 import logging
-import asyncio
 from argparse import ArgumentParser
 
-from gitutils.pull import find_remote
+from gitutils.pull import coro_remote
+from gitutils.runner import runner
 
 MODE = 'fetch'
 
@@ -23,19 +21,8 @@ def main():
     if P.verbose:
         logging.basicConfig(level=logging.DEBUG)
 
-    if os.name == 'nt' and (3, 7) <= sys.version_info < (3, 8):
-        asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy())
-
-    if sys.version_info >= (3, 7):
-        asyncio.run(find_remote(MODE, P.codepath))
-    else:
-        if os.name == 'nt':
-            loop = asyncio.ProactorEventLoop()
-        else:
-            loop = asyncio.new_event_loop()
-            asyncio.get_child_watcher().attach_loop(loop)
-        loop.run_until_complete(find_remote(MODE, P.codepath))
-        loop.close()
+    remotes = runner(coro_remote, MODE, P.codepath)
+    print('\n'.join(map(str, remotes)))
 
 
 if __name__ == '__main__':

find_missing_file.py

@@ -5,7 +5,7 @@
 """
 from pathlib import Path
 from sys import stderr
-from gitutils import find_dir_missing_file
+from gitutils.find import find_dir_missing_file
 from argparse import ArgumentParser
 
 

gitemail.py

@@ -19,7 +19,7 @@
 from typing import Sequence
 from argparse import ArgumentParser
 
-from gitutils import gitemail
+from gitutils.email import gitemail
 from gitutils.git import MAGENTA, BLACK
 
 cwd = Path(__file__).parent

gitutils/__init__.py

@@ -3,45 +3,5 @@
 Speed is emphasized throughout, with pipelining and concurrent `asyncio` routines throughout
 for fastest operation on large numbers of repos.
 """
-from pathlib import Path
-from typing import List
-import shutil
 
-from .git import listchanged  # noqa: F401
-from .branch import findbranch  # noqa: F401
-from .email import gitemail  # noqa: F401
-from .pull import fetchpull  # noqa: F401
-
-
-def find_dir_missing_file(fn: str, path: Path, copyfile: Path = None) -> List[Path]:
-    """
-    if directory is missing a file, copy the file to that directory
-
-    Parameters
-    ----------
-    fn : pathlib.Path
-        filename to look for
-    path : pathlib.Path
-        top-level directory to check directories under
-    copyfile : pathlib.Path, optional
-        if present, copy this file into the directory that doesn't have it
-
-    Results
-    -------
-    missing : list of pathlib.Path
-        directories that were missing the file and it wasn't copied there.
-    """
-    path = Path(path).expanduser()
-
-    dlist = (x for x in path.iterdir() if x.is_dir())
-
-    missing: List[Path] = []
-    for d in dlist:
-        if not (d / fn).is_file():
-            if isinstance(copyfile, Path):
-                shutil.copy(copyfile, d)
-                print('copied', copyfile, 'to', d)
-            else:
-                missing.append(d)
-
-    return missing
+from .runner import runner  # noqa: F401

gitutils/branch.py

@@ -5,9 +5,10 @@
 https://stackoverflow.com/a/45028375
 """
 
-from typing import AsyncGenerator, Tuple
+import typing
 from pathlib import Path
 import asyncio
+import logging
 
 from .git import GITEXE, gitdirs
 
@@ -17,20 +18,20 @@
 BRANCH_SIMPLE = ['branch', '--show-current']  # Git >= 2.22
 
 
-async def findbranch(mainbranch: str, rdir: Path) -> AsyncGenerator[Tuple[str, str], None]:
+async def different_branch(mainbranch: str, path: Path) -> typing.Tuple[str, str]:
     """
-    find all branches in tree not matching "mainbranch"
+    does branch not match "mainbranch"
 
     Parameters
     ----------
 
     mainbranch : str
         branch name that's "normal" e.g. master
-    rdir : pathlib.Path
-        top-level directory to work under e.g. ~/code/
+    path : pathlib.Path
+        Git repo to check
 
-    Yields
-    ------
+    Returns
+    -------
 
     branch : tuple of pathlib.Path, str
         repo path and branch name
@@ -39,16 +40,18 @@ async def findbranch(mainbranch: str, rdir: Path) -> AsyncGenerator[Tuple[str, s
     https://docs.python.org/3/library/asyncio-subprocess.html#subprocess-and-threads
     """
 
-    rdir = Path(rdir).expanduser()
+    proc = await asyncio.create_subprocess_exec(*[GITEXE, '-C', str(path)] + BRANCH_REV,
+                                                stdout=asyncio.subprocess.PIPE)
+    stdout, _ = await proc.communicate()
+    logging.info(str(path))
 
-    for d in gitdirs(rdir):
-        proc = await asyncio.create_subprocess_exec(*[GITEXE, '-C', str(d)] + BRANCH_REV,
-                                                    stdout=asyncio.subprocess.PIPE)
-        stdout, _ = await proc.communicate()
+    branchname = stdout.decode('utf8').rstrip()
 
-        branchname = stdout.decode('utf8').rstrip()
+    if mainbranch != branchname:
+        return path.name, branchname
+    return None
 
-        if mainbranch in branchname:
-            continue
 
-        yield d.name, branchname
+async def coro_local(branch: str, path: Path) -> typing.List[Path]:
+    futures = [different_branch(branch, d) for d in gitdirs(path)]
+    return list(filter(None, await asyncio.gather(*futures)))

gitutils/find.py

@@ -0,0 +1,39 @@
+from pathlib import Path
+import typing
+import shutil
+
+
+def find_dir_missing_file(fn: str, path: Path, copyfile: Path = None) -> typing.List[Path]:
+    """
+    if directory is missing a file, copy the file to that directory
+
+    Parameters
+    ----------
+    fn : str
+        filename to look for
+    path : pathlib.Path
+        top-level directory to check directories under
+    copyfile : pathlib.Path, optional
+        if present, copy this file into the directory that doesn't have it
+
+    Results
+    -------
+    missing : list of pathlib.Path
+        directories that were missing the file and it wasn't copied there.
+    """
+    path = Path(path).expanduser()
+    if not path.is_dir():
+        raise NotADirectoryError(path)
+
+    dlist = (x for x in path.iterdir() if x.is_dir())
+
+    missing = []  # type: typing.List[Path]
+    for d in dlist:
+        if not (d / fn).is_file():
+            if isinstance(copyfile, Path):
+                shutil.copy(copyfile, d)
+                print('copied', copyfile, 'to', d)
+            else:
+                missing.append(d)
+
+    return missing

gitutils/git.py

@@ -3,7 +3,7 @@
 """
 from pathlib import Path
 import subprocess
-from typing import List, Iterator, Any
+import typing
 import shutil
 
 try:
@@ -15,9 +15,9 @@
     MAGENTA = BLACK = ''
 
 TIMEOUT = 30.  # arbitrary, seconds
-GITEXE: Any = shutil.which('git')  # type: ignore
+GITEXE = shutil.which('git')  # type: str
 if not GITEXE:
-    raise FileNotFoundError('Could not find executable for Git')
+    raise ImportError('Could not find executable for Git')
 
 """
 replaced by git status --porcelain:
@@ -28,7 +28,7 @@
 """
 
 
-def gitdirs(path: Path) -> Iterator[Path]:
+def gitdirs(path: Path) -> typing.Iterator[Path]:
     """
     Generator for Git directories
 
@@ -77,13 +77,13 @@ def baddir(path: Path) -> bool:
 
     try:
         bad = (path / '.nogit').is_file() or not (path / '.git' / 'HEAD').is_file()
-    except PermissionError:
+    except PermissionError:  # Windows
         bad = True
 
     return bad
 
 
-def listchanged(path: Path) -> List[str]:
+def listchanged(path: Path) -> typing.List[str]:
     """very quick check if any files were modified in this Git repo
 
     Parameters