Move templates to remote repo (#131)

* move template to extra repo

* updated docs

* adjust list-templates to git remotes

* use contextmanager and tmpfile to clean up repositories

* python38

* fix incorrect exception handling

* python 3.9 fix; doc string

* remove misleading line

Author: n-gao

docs.md

@@ -414,6 +414,8 @@ $ seml project init [OPTIONS] [DIRECTORY]
 * `-n, --name TEXT`: The name of the project. (By default inferred from the directory name.)
 * `-u, --username TEXT`: The author name to use for the project. (By default inferred from $USER)
 * `-m, --usermail TEXT`: The author email to use for the project. (By default empty.)
+* `-r, --git-remote TEXT`: The git remote to use for the project. (By default SETTINGS.TEMPLATE_REMOTE.)
+* `-c, --git-commit TEXT`: The exact git commit to use. May also be a tag or branch (By default latest)
 * `-y, --yes`: Automatically confirm all dialogues with yes.
 * `--help`: Show this message and exit.
 
@@ -429,6 +431,8 @@ $ seml project list-templates [OPTIONS]
 
 **Options**:
 
+* `-r, --git-remote TEXT`: The git remote to use for the project. (By default SETTINGS.TEMPLATE_REMOTE.)
+* `-c, --git-commit TEXT`: The exact git commit to use. May also be a tag or branch (By default latest)
 * `--help`: Show this message and exit.
 
 ## `seml reload-sources`

src/seml/__main__.py

@@ -1038,21 +1038,64 @@ def init_project_command(
             help='The author email to use for the project. (By default empty.)',
         ),
     ] = None,
+    git_remote: Annotated[
+        str,
+        typer.Option(
+            '-r',
+            '--git-remote',
+            help='The git remote to use for the project. (By default SETTINGS.TEMPLATE_REMOTE.)',
+        ),
+    ] = None,
+    git_commit: Annotated[
+        str,
+        typer.Option(
+            '-c',
+            '--git-commit',
+            help='The exact git commit to use. May also be a tag or branch (By default latest)',
+        ),
+    ] = None,
     yes: YesAnnotation = False,
 ):
     """
     Initialize a new project in the given directory.
     """
-    init_project(directory, project_name, user_name, user_mail, template, yes)
+    init_project(
+        directory,
+        project_name,
+        user_name,
+        user_mail,
+        template,
+        git_remote,
+        git_commit,
+        yes,
+    )
 
 
 @app_project.command('list-templates')
 @restrict_collection(False)
-def list_templates_command(ctx: typer.Context):
+def list_templates_command(
+    ctx: typer.Context,
+    git_remote: Annotated[
+        str,
+        typer.Option(
+            '-r',
+            '--git-remote',
+            help='The git remote to use for the project. (By default SETTINGS.TEMPLATE_REMOTE.)',
+        ),
+    ] = None,
+    git_commit: Annotated[
+        str,
+        typer.Option(
+            '-c',
+            '--git-commit',
+            help='The exact git commit to use. May also be a tag or branch (By default latest)',
+        ),
+    ] = None,
+):
     """
     List available project templates.
     """
-    print_available_templates()
+    print_available_templates(git_remote, git_commit)
 
 
 @dataclass

src/seml/project.py

@@ -1,7 +1,10 @@
 import logging
 import os
+from contextlib import contextmanager
 from pathlib import Path
-from typing import Optional, Union
+from typing import List, Optional, Union
+
+from seml.settings import SETTINGS
 
 
 def init_project(
@@ -10,6 +13,8 @@ def init_project(
     user_name: Optional[str] = None,
     user_mail: Optional[str] = None,
     template: str = 'default',
+    git_remote: Optional[str] = None,
+    git_commit: Optional[str] = None,
     yes: bool = False,
 ):
     """
@@ -27,16 +32,22 @@ def init_project(
         The email of the user. If not given, ''.
     template : str
         The template to use for the project.
+    git_repo : Optional[str]
+        The URL of the git repository to use.
+    git_commit : Optional[str]
+        The commit to use.
+    git_branch : Optional[str]
+        The branch to use.
     yes : bool
         If True, no confirmation is asked before initializing the project.
     """
-    import importlib.resources
     from click import prompt
     from gitignore_parser import parse_gitignore
 
     if directory is None:
         directory = Path()
     directory = Path(directory).absolute()
+
     # Ensure that the directory exists
     if not directory.exists():
         directory.mkdir(parents=True)
@@ -48,84 +59,132 @@ def init_project(
             type=bool,
         ):
             exit(1)
-    logging.info(f'Initializing project in "{directory}" using template "{template}"')
-
-    template_path = (
-        Path(str(importlib.resources.files('seml')))
-        / 'templates'
-        / 'project'
-        / template
-    )
-    if not template_path.exists():
-        logging.error(f'Template "{template}" does not exist')
-        exit(1)
-
-    if project_name is None:
-        project_name = directory.name
-    if user_name is None:
-        user_name = os.getenv('USER', os.getenv('USERNAME', 'user'))
-    if user_mail is None:
-        user_mail = 'my@mail.com'
-    format_map = dict(
-        project_name=project_name, user_name=user_name, user_mail=user_mail
-    )
-
-    gitignore_path = template_path / '.gitignore'
-    if gitignore_path.exists():
-        ignore_file = parse_gitignore(gitignore_path)
-    else:
-
-        def ignore_file(file_path: str):
-            return False
-
-    # Copy files one-by-one
-    for src in template_path.glob('**/*'):
-        # skip files ignored by .gitignore
-        if ignore_file(str(src)):
-            continue
-        # construct destination
-        file_name = src.relative_to(template_path)
-        target_file_name = Path(str(file_name).format_map(format_map))
-        dst = directory / target_file_name
-        # Create directories
-        if src.is_dir():
-            if not dst.exists():
-                dst.mkdir()
-        elif not dst.exists():
-            # For templates fill in variables
-            if src.suffix.endswith('.template'):
-                dst = dst.with_suffix(src.suffix.removesuffix('.template'))
-                dst.write_text(src.read_text().format_map(format_map))
-            else:
-                # Other files copy directly
-                dst.write_bytes(src.read_bytes())
+
+    tmp_dir = checkout_template_repo(git_remote, git_commit)
+    with checkout_template_repo(git_remote, git_commit) as tmp_dir:
+        template_path = tmp_dir / 'templates' / template
+        if not template_path.exists():
+            logging.error(f'Template "{template}" does not exist')
+            exit(1)
+
+        logging.info(
+            f'Initializing project in "{directory}" using template "{template}".'
+        )
+
+        if project_name is None:
+            project_name = directory.name
+        if user_name is None:
+            user_name = os.getenv('USER', os.getenv('USERNAME', 'user'))
+        if user_mail is None:
+            user_mail = 'my@mail.com'
+        format_map = dict(
+            project_name=project_name, user_name=user_name, user_mail=user_mail
+        )
+
+        gitignore_path = template_path / '.gitignore'
+        if gitignore_path.exists():
+            ignore_file = parse_gitignore(gitignore_path)
+        else:
+
+            def ignore_file(file_path: str):
+                return False
+
+        # Copy files one-by-one
+        for src in template_path.glob('**/*'):
+            # skip files ignored by .gitignore
+            if ignore_file(str(src)):
+                continue
+            # construct destination
+            file_name = src.relative_to(template_path)
+            target_file_name = Path(str(file_name).format_map(format_map))
+            dst = directory / target_file_name
+            # Create directories
+            if src.is_dir():
+                if not dst.exists():
+                    dst.mkdir()
+            elif not dst.exists():
+                # For templates fill in variables
+                if src.suffix.endswith('.template'):
+                    dst = dst.with_suffix(src.suffix.replace('.template', ''))
+                    dst.write_text(src.read_text().format_map(format_map))
+                else:
+                    # Other files copy directly
+                    dst.write_bytes(src.read_bytes())
     logging.info('Project initialized successfully')
 
 
-def get_available_templates():
+@contextmanager
+def checkout_template_repo(
+    git_remote: Optional[str] = None, git_commit: Optional[str] = None
+):
+    """
+    Context manager to clone the template repository. The cloned repository
+    is deleted after the context is left.
+
+    Parameters
+    ----------
+    git_remote : Optional[str]
+        The git remote to use.
+    git_commit : Optional[str]
+        The git commit to use.
+    """
+    import tempfile
+    from git import Repo
+
+    if git_remote is None:
+        git_remote = SETTINGS.TEMPLATE_REMOTE
+
+    with tempfile.TemporaryDirectory(dir=SETTINGS.TMP_DIRECTORY) as tmp_dir:
+        try:
+            repo = Repo.clone_from(git_remote, tmp_dir)
+            if git_commit is not None:
+                repo.head.reference = repo.commit(git_commit)
+                repo.head.reset(index=True, working_tree=True)
+        except Exception as e:
+            logging.error(
+                f'Failed to clone git repository "{git_remote}" to "{tmp_dir}"'
+            )
+            logging.error(e)
+            exit(1)
+        yield Path(repo.working_dir)
+
+
+def get_available_templates(
+    git_remote: Optional[str] = None, git_commit: Optional[str] = None
+) -> List[str]:
     """
     Return a list of available templates.
 
+    Parameters
+    ----------
+    git_remote : Optional[str]
+        The git remote to use.
+    git_commit : Optional[str]
+        The git commit to use.
+
     Returns
     -------
     List[str]
         A list of available templates.
     """
-    import importlib.resources
+    with checkout_template_repo(git_remote, git_commit) as repo:
+        return [template.name for template in (repo / 'templates').iterdir()]
 
-    return [
-        template.name
-        for template in (
-            Path(str(importlib.resources.files('seml'))) / 'templates' / 'project'
-        ).iterdir()
-    ]
 
-
-def print_available_templates():
+def print_available_templates(
+    git_remote: Optional[str] = None, git_commit: Optional[str] = None
+):
     """
     Print the available templates.
+
+    Parameters
+    ----------
+    git_remote : Optional[str]
+        The git remote to use.
+    git_commit : Optional[str]
+        The git commit to use.
     """
     result = 'Available templates:'
-    for template in get_available_templates():
+    for template in get_available_templates(git_remote, git_commit):
         result += f'\n  - {template}'
     logging.info(result)

src/seml/settings.py

@@ -19,6 +19,7 @@
         # Directory which is used on the compute nodes to dump scripts and Python code.
         # Only change this if you know what you're doing.
         'TMP_DIRECTORY': '/tmp',
+        'TEMPLATE_REMOTE': 'https://github.com/TUM-DAML/seml-templates.git',
         'DATABASE': {
             # location of the MongoDB config. Default: $HOME/.config/seml/monogdb.config
             'MONGODB_CONFIG_PATH': APP_DIR / 'mongodb.config'