SSH support, YAML instead of JSON

Author: desertkun

Readme.md

@@ -1,29 +1,53 @@
 PyPiGit
 =======
 
-A simple PyPi-like server that uses git repositories as a source of python packages.
+A simple PyPi-like server that automatically generates python packages from git tags.
 
 How To
 ------
 
-* Prepare `repos.json`:
-```json
-{
-    "repositories": [
-        "https://github.com/<author>/<repo>.git",
-        "git@github.com:<author>/<repo>.git"
-    ]
-}
+* Prepare `repos.yaml`:
+```yaml
+repositories:
+  - https://github.com/<author>/<repo>.git
+  - git@github.com:<author>/<repo>.git
 ```
 
-* Start a server:
+* Start a server (see `python -m pypigit.server --help` for all arguments):
 ```bash
-python -m pypigit.server --repos="repos.json"
+python -m pypigit.server --repos="repos.yaml"
 ```
 
 * Draft a release (that will create certain tag)
+* Install a package with pip:
 ```bash
 pip install --extra-index-url http://localhost:9498/simple <repo>==1.0.2
 ```
 
-It will automatically clone tag `1.0.2`, build a `sdist` and cache it.
+It will automatically build a package from tag `1.0.2`, and deliver it to pip. Once
+built, it will be stored in `--cache-direcory` so consecutive calls will take no time.
+
+SSH
+---
+
+This project is aimed to host private packages, so ssh git remotes are supported:
+```yaml
+repositories:
+  - url: git@github.com:<author>/<repo>.git
+    ssh_key: |
+      -----BEGIN RSA PRIVATE KEY-----
+      <private key>
+      -----END RSA PRIVATE KEY-----
+```
+If you have same private ssh key for all repositories, you can simply define `default_ssh_key` instead:
+```yaml
+repositories:
+  - git@github.com:<author>/<repo>.git
+  - git@github.com:<author>/<repo>.git
+  - git@github.com:<author>/<repo>.git
+default_ssh_key: |
+  -----BEGIN RSA PRIVATE KEY-----
+  <private key>
+  -----END RSA PRIVATE KEY-----
+    
+```

pypigit/__init__.py

@@ -1,5 +1,7 @@
 
 from tornado.ioloop import IOLoop
+from tempfile import mktemp
+import os
 
 
 def run_on_executor(method):
@@ -8,3 +10,51 @@ def wrapper(self, *args):
         return IOLoop.current().run_in_executor(executor, method, self, *args)
     return wrapper
 
+
+class PrivateSSHKeyContext(object):
+    """
+    This context manager class creates temporary file with ssh_private_key in it,
+        and conveniently returns path to it, taking care to remove the file afterwards:
+
+    with PrivateSSHKeyWrapper("private ssh key string") as key_path:
+        use key_path here for ssh operations
+
+    key_path deleted afterwards
+
+    """
+
+    def __init__(self, ssh_private_key=None):
+        self.name = None
+        self.ssh_private_key = ssh_private_key
+        self.sys_fd = None
+
+    @staticmethod
+    def convert_path(path):
+        separator = os.path.sep
+        if separator != '/':
+            path = path.replace(os.path.sep, '/')
+        return path
+
+    def __enter__(self):
+        if self.ssh_private_key is None:
+            return None
+
+        self.name = mktemp()
+
+        with open(self.name, 'w') as f:
+            f.write(self.ssh_private_key)
+            f.write("\n")
+
+        return PrivateSSHKeyContext.convert_path(self.name)
+
+    def __exit__(self, exc, value, tb):
+        self.cleanup()
+
+    def cleanup(self):
+        if self.name is None:
+            return
+
+        try:
+            os.remove(self.name)
+        except IOError:
+            pass

pypigit/handler.py

@@ -1,5 +1,6 @@
 
 from tornado.web import RequestHandler, HTTPError
+from . repos import GitRepositoryError
 
 
 class MainHandler(RequestHandler):
@@ -20,7 +21,10 @@ async def get(self, package_name):
         if repo is None:
             raise HTTPError(404, "No such package")
 
-        versions = await repo.list_versions()
+        try:
+            versions = await repo.list_versions()
+        except GitRepositoryError as e:
+            raise HTTPError(e.code, e.message)
 
         self.render("templates/package_versions.html", package_name=package_name, versions=versions)
 

pypigit/repos.py

@@ -1,27 +1,56 @@
 
-from . import run_on_executor
+from . import run_on_executor, PrivateSSHKeyContext
+
 import giturlparse
-import json
 from concurrent.futures import ThreadPoolExecutor
 import git
 import os
 import os.path
 import subprocess
 import stat
+import yaml
+
 from shutil import copyfile, rmtree
 from tempfile import TemporaryDirectory
 
 
+def git_ssh_command(private_key):
+    return "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i {0}".format(private_key)
+
+
+def git_ssh_environment(g, ssh_private_key_filename=None):
+    if ssh_private_key_filename:
+        return g.custom_environment(GIT_SSH_COMMAND=git_ssh_command(ssh_private_key_filename))
+    return g.custom_environment()
+
+
 class CacheRedirectException(Exception):
     def __init__(self, url):
         self.url = url
 
 
+class GitRepositoryError(Exception):
+    def __init__(self, code, message):
+        self.code = code
+        self.message = message
+
+
 class GitRepository(object):
     executor = ThreadPoolExecutor()
-    g = git.cmd.Git()
 
-    def __init__(self, url, cache_directory, public_url):
+    def __init__(self, settings, cache_directory, public_url, default_private_key=None):
+
+        if isinstance(settings, str):
+            url = settings
+            self.private_key = None
+        elif isinstance(settings, dict):
+            url = settings.get("url", None)
+            if url is None:
+                raise RuntimeError("Repo is an object but has no `url` property")
+            self.private_key = settings.get("ssh_key", default_private_key)
+        else:
+            raise RuntimeError("Repo should be a string or a object")
+
         self.repo_url = url
         self.parsed_url = giturlparse.parse(url)
         if not self.parsed_url.valid:
@@ -33,14 +62,20 @@ def __init__(self, url, cache_directory, public_url):
     @run_on_executor
     def list_versions(self):
         repo_name = self.name()
-        tags = GitRepository.g.ls_remote(self.repo_url, tags=True)
-        result = {}
-        for line in tags.split('\n'):
-            ref_hash, ref = line.split('\t')
-            tag_name = ref.split("/")[-1]
-            tar_name = self.package_tar(tag_name)
-            result[tar_name] = self.public_url + "/download/" + repo_name + "/" + tar_name
-        return result
+        with PrivateSSHKeyContext(ssh_private_key=self.private_key) as ssh_private_key_filename:
+            g = git.cmd.Git()
+            with git_ssh_environment(g, ssh_private_key_filename=ssh_private_key_filename):
+                try:
+                    tags = g.ls_remote(self.repo_url, tags=True)
+                except git.GitCommandError as e:
+                    raise GitRepositoryError(500, "Failed to fetch remote repository: {0}".format(e.status))
+                result = {}
+                for line in tags.split('\n'):
+                    ref_hash, ref = line.split('\t')
+                    tag_name = ref.split("/")[-1]
+                    tar_name = self.package_tar(tag_name)
+                    result[tar_name] = self.public_url + "/download/" + repo_name + "/" + tar_name
+                return result
 
     def get_cache_url(self, package_version):
         return self.cache_public_url + "/" + self.package_tar(package_version)
@@ -59,20 +94,26 @@ def download(self, package_version):
         if os.path.isfile(os.path.join(self.cache_directory, tar_name)):
             return cache_url
 
+        # noinspection PyUnusedLocal
         def set_rw(operation, name, exc):
             os.chmod(name, stat.S_IWRITE)
             os.remove(name)
             return True
 
         with TemporaryDirectory(prefix="pypigit") as temp_dir:
-            r = git.Git(temp_dir)
-            r.clone(self.repo_url, branch=package_version, depth=1)
+            g = git.Git(temp_dir)
+
+            with PrivateSSHKeyContext(ssh_private_key=self.private_key) as ssh_private_key_filename:
+                with git_ssh_environment(g, ssh_private_key_filename=ssh_private_key_filename):
+                    g.clone(self.repo_url, branch=package_version, depth=1)
+
             build = os.path.join(temp_dir, repo_name)
             p = subprocess.Popen("python setup.py sdist", stdout=subprocess.PIPE, shell=True, cwd=build)
             p.communicate()
 
             copyfile(os.path.join(build, "dist", tar_name), os.path.join(self.cache_directory, tar_name))
 
+            # noinspection PyTypeChecker
             rmtree(build, onerror=set_rw)
 
         return cache_url
@@ -85,18 +126,23 @@ class GitRepositories(object):
     def __init__(self, repos_filename, cache_directory, public_url):
 
         with open(repos_filename, "r") as f:
-            repos = json.load(f)
+            repos = yaml.load(f)
 
         if not isinstance(repos, dict):
             raise RuntimeError("--repos file should be a json object")
 
+        default_private_key = repos.get("default_ssh_key")
+
         if "repositories" not in repos:
             raise RuntimeError("No 'repositories' section in --repos file")
 
         self.repositories = {
             repo.name(): repo
-            for repo in map(lambda path: GitRepository(path, cache_directory, public_url),
-                            filter(lambda s: isinstance(s, str), repos["repositories"]))
+            for repo in map(
+                lambda settings: GitRepository(
+                    settings, cache_directory, public_url, default_private_key=default_private_key
+                ), repos["repositories"]
+            )
         }
 
     def find(self, name):

pypigit/server.py

@@ -11,7 +11,7 @@
 define("port", type=int, default="9498", help="Listen port")
 define("cache-directory", type=str, default=".pypigit", help="Cache directory")
 define("public-url", type=str, default="http://localhost:9498", help="Public URL directory")
-define("repos", type=str, help="JSON file with list of git remotes")
+define("repos", type=str, help="YAML file with list of git remotes")
 
 
 class PyPiGITServer(Application):

setup.py

@@ -3,17 +3,18 @@
 
 setup(
     name='pypigit',
-    version='0.1.0',
-    description='A simple PyPi-like server that uses git repositories as a source of python packages',
+    version='0.2',
+    description='A simple PyPi-like server that automatically generates python packages from git tags',
     author='desertkun',
     license='MIT',
     author_email='desertkun@gmail.com',
     url='https://github.com/anthill-utils/pypigit',
     packages=find_packages(),
     zip_safe=False,
     install_requires=[
-        "tornado==5.1.1",
-        "GitPython==2.1.7",
-        "giturlparse.py==0.0.5"
+        "tornado>=5.0",
+        "GitPython>=2.1.7",
+        "giturlparse.py>=0.0.5",
+        "PyYAML>=3.13"
     ]
 )