Update MySQL/MariaDB stuff

Author: apdavison

doc/authors.txt

@@ -31,6 +31,7 @@ current affiliation of a contributor.
 * Dan Padilha [18]
 * Simon Chabot [19]
 * @maharjun
+* David Kleiven [20]
 
 1. Unité de Neuroscience, Information et Complexité, CNRS UPR 3293, Gif-sur-Yvette, France
 2. Ecole Centrale de Lille, Lille, France
@@ -51,6 +52,7 @@ current affiliation of a contributor.
 17. Institute of Nanotechnology, Karlsruhe Institute of Technology, Germany
 18. Shoal Engineering Pty Ltd, Canberra, Australia
 19. CEREMA Direction territoriale Méditerranée, Nice, France
+20. NTNU, Trondheim, Norway
 
 If we've somehow missed you off the list I'm very sorry - please let us know.
 

pyproject.toml

@@ -63,6 +63,8 @@ remote = ["httplib2"]
 
 postgres = ["psycopg2-binary"]
 
+mysql = ["mysqlclient"]
+
 latex = ["jinja2"]
 
 sphinx = ["docutils"]

sumatra/commands.py

@@ -40,7 +40,14 @@
          "repeat", "diff", "help", "export", "upgrade", "sync", "migrate", "version",
          "view")
 
-store_arg_help = "The argument can take the following forms: (1) `/path/to/sqlitedb` - DjangoRecordStore is used with the specified Sqlite database, (2) `http[s]://location` - remote HTTPRecordStore is used with a remote Sumatra server, (3) `postgres://username:password@hostname/databasename` - DjangoRecordStore is used with specified Postgres database."
+store_arg_help = (
+    "The argument can take the following forms: "
+    "(1) `/path/to/sqlitedb` - DjangoRecordStore is used with the specified Sqlite database, "
+    "(2) `http[s]://location` - remote HTTPRecordStore is used with a remote Sumatra server, "
+    "(3) `postgres://username:password@hostname/databasename` - DjangoRecordStore is used with specified PostgreSQL database."
+    "(4) `mysql://username:password@hostname/databasename` - DjangoRecordStore is used with specified MySQL database."
+    "(5) `mariadb://username:password@hostname/databasename` - DjangoRecordStore is used with specified MariaDB database."
+)
 
 ## recommended method for modifying warning formatting
 ## see https://docs.python.org/2/library/warnings.html#warnings.showwarning

sumatra/recordstore/django_store/__init__.py

@@ -349,7 +349,7 @@ def _dump(self, indent=2):
 
     @classmethod
     def accepts_uri(cls, uri):
-        return uri[:8] == "postgres" or uri.startswith(('mysql', 'mariadb')) or os.path.exists(uri) or os.path.exists(uri + ".db")
+        return uri.startswith(("postgres", "mysql", "mariadb")) or os.path.exists(uri) or os.path.exists(uri + ".db")
 
     def backup(self):
         """

test/system/fixtures/Dockerfile.mysql

@@ -1,120 +1,90 @@
 """
-Tests using a MySQL-based record store.
+Tests using a MariaDB/MySQL-based record store.
 
-
-Usage:
-    nosetests -v test_mysql.py
-or:
-    python test_mysql.py
 """
-from __future__ import print_function
-from __future__ import unicode_literals
-from future import standard_library
-standard_library.install_aliases()
-from builtins import input
 
 import os
-from urllib.parse import urlparse
+import shutil
+import tempfile
 try:
     import docker
-    if "DOCKER_HOST" in os.environ:
-        have_docker = True
-    else:
-        have_docker = False
+    have_docker = True
 except ImportError:
     have_docker = False
-from unittest import SkipTest
-import utils
-from utils import setup, teardown as default_teardown, run_test, build_command
+try:
+    import mysqlclient
+    have_mysql = True
+except ImportError:
+    have_mysql = False
 
-ctr = None
-dkr = None
-image = "mysql_test"
 
+from utils import run_test, build_command
 
-def create_script():
-    with open(os.path.join(utils.working_dir, "main.py"), "w") as fp:
-        fp.write("print('Hello world!')\n")
+import pytest
 
+DOCKER_IMAGE = "mariadb:latest"
 
-def get_url():
-    info = dkr.containers()[0]
-    assert info["Image"] == image
-    host = urlparse(dkr.base_url).hostname
-    return "{0}:{1}".format(host, info["Ports"][0]["PublicPort"])
 
+def get_url(ctr):
+    ctr.reload()  # required to get auto-assigned ports
+    info = ctr.ports["3306/tcp"][0]
+    return f"{info['HostIp']}:{info['HostPort']}"
 
-def start_pg_container():
-    """
-    Launch a Docker container running MySQL, return the URL.
 
-    Requires the "mysql_test" image. If this does not yet exist, run
+@pytest.fixture(scope="module")
+def mysql_container():
+    """
+    Launch a Docker container running MariaDB, return the URL.
 
-        docker build -t mysql_test - < fixtures/Dockerfile.mysql
     """
-    global ctr, dkr
-    env = docker.utils.kwargs_from_env(assert_hostname=False)
-    dkr = docker.Client(timeout=60, **env)
-
-    host_config = dkr.create_host_config(publish_all_ports=True)
-    # docker run -rm -P -name ms_test mysql_test
-    ctr = dkr.create_container(image, command=None, hostname=None, user=None,
-                               detach=False, stdin_open=False, tty=False,
-                               ports=None, environment=None, dns=None,
-                               volumes=None, volumes_from=None,
-                               network_disabled=False, name=None,
-                               entrypoint=None, cpu_shares=None,
-                               working_dir=None, host_config=host_config)
-    dkr.start(ctr)
-    utils.env["url"] = get_url()
-
-
-def teardown():
-    global ctr, dkr
-    default_teardown()
-    if dkr is not None:
-        dkr.stop(ctr)
-
-
-test_steps = [
-    create_script,
-    ("Create repository", "git init"),
-    ("Add main file", "git add main.py"),
-    ("Commit main file", "git commit -m 'initial'"),
-    start_pg_container,
-    ("Set up a Sumatra project",
-     build_command("smt init --store=mysql://docker:docker@{}/sumatra_test -m main.py -e python MyProject", "url")),
-    ("Show project configuration", "smt info"),  # TODO: add assert
-    ("Run a computation", "smt run")
-]
-
-
-# TODO: add test skips where docker not found
-
-
-def test_all():
-    """Test generator for Nose."""
-    if not have_docker:
-        raise SkipTest("Tests require docker")
+    dkr = docker.from_env()
+
+    ctr = dkr.containers.run(
+        DOCKER_IMAGE, detach=True, publish_all_ports=True,
+        environment={
+            "MARIADB_ROOT_PASSWORD": "the_root_password",
+            "MARIADB_DATABASE": "sumatra_test",
+            "MARIADB_USER": "docker",
+            "MARIADB_PASSWORD": "docker"
+        }
+    )
+    container_url = get_url(ctr)
+    yield container_url
+    ctr.stop()
+
+
+@pytest.mark.skipif(not have_docker, reason="Tests require docker Python package")
+@pytest.mark.skipif(not have_mysql, reason="Tests require mysqlclient")
+def test_all(mysql_container):
+    """Run a series of Sumatra commands"""
+    temporary_dir = os.path.realpath(tempfile.mkdtemp())
+    working_dir = os.path.join(temporary_dir, "sumatra_exercise")
+    os.mkdir(working_dir)
+    env = {
+        "labels": [],
+        "working_dir": working_dir,
+        "url": mysql_container
+    }
+
+    def create_script():
+        with open(os.path.join(working_dir, "main.py"), "w") as fp:
+            fp.write("print('Hello world!')\n")
+
+    test_steps = [
+        create_script,
+        ("Create repository", "git init"),
+        ("Add main file", "git add main.py"),
+        ("Commit main file", "git commit -m 'initial'"),
+        ("Set up a Sumatra project",
+        build_command("smt init --store=mysql://docker:docker@{}/sumatra_test -m main.py -e python MyProject", "url")),
+        #"smt init -m main.py -e python MyProject"),
+        ("Show project configuration", "smt info"),  # TODO: add assert
+        ("Run a computation", "smt run")
+    ]
     for step in test_steps:
         if callable(step):
             step()
         else:
-            run_test.description = step[0]
-            yield tuple([run_test] + list(step[1:]))
+            run_test(*step[1:], env=env)
 
-
-if __name__ == '__main__':
-    # Run the tests without using Nose.
-    setup()
-    for step in test_steps:
-        if callable(step):
-            step()
-        else:
-            print(step[0])  # description
-            run_test(*step[1:])
-    response = input("Do you want to delete the temporary directory (default: yes)? ")
-    if response not in ["n", "N", "no", "No"]:
-        teardown()
-    else:
-        print("Temporary directory %s not removed" % utils.temporary_dir)
+    shutil.rmtree(temporary_dir)