dump MySQL to AWS S3

.github/workflows/continuous-integration-pip.yml

@@ -106,7 +106,7 @@ jobs:
       - name: Run coverage test
         if: startsWith(matrix.os, 'ubuntu')
         env:
-          SKIP_ENDPOINT_TEST:   ${{ vars.SKIP_ENDPOINT_TEST }}
+          SKIP_ENDPOINT_TEST: ${{ vars.SKIP_ENDPOINT_TEST }}
         run: |
           CHROME_PATH=${{ steps.setup-chrome.outputs.chrome-path }} make pytest-coverage
 
@@ -124,5 +124,4 @@ jobs:
 
       - name: Show files in directory
         run: |
-          make coverage
           ls -l

dbutil.py

@@ -17,7 +17,7 @@
 
 if __name__ == "__main__":
     import argparse
-    parser = argparse.ArgumentParser(description="Database Maintenance Program. The database to act upon is specified in the ini file specified by the PLANTTRACER_CREDENTIALS environment variable, in the sections for [dbreader] and [dbwriter]",
+    parser = argparse.ArgumentParser(description="Database Maintenance Program. The database to act upon is specified in the ini file with the --readconfig or --rootconfig",
                                      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
     required = parser.add_argument_group('required arguments')
     required.add_argument(
@@ -48,7 +48,8 @@
     parser.add_argument("--freshen",help="Non-destructive cleans up the movie metadata for all movies.",action='store_true')
     parser.add_argument("--clean",help="Destructive cleans up the movie metadata for all movies.",action='store_true')
     parser.add_argument("--schema", help="Specify schema file to use", default=paths.SCHEMA_FILE)
-    parser.add_argument("--dump", help="Backup all objects as JSON files and movie files to new directory called DUMP.  ")
+    parser.add_argument("--dump", help="Dump all objects as JSON files and movie files to new directory called DUMP.")
+    parser.add_argument("--sqlbackup", help="Backup the MySQL database to a single SQL file")
     parser.add_argument("--add_admin", help="Add --admin_email user as a course admin to the course specified by --course_id, --course_name, or --course_name", action='store_true')
     parser.add_argument("--course_id", help="integer course id", type=int)
     parser.add_argument("--course_key", help="integer course id")
@@ -236,6 +237,17 @@
         dbmaint.freshen(True)
         sys.exit(0)
 
+    # These need a database config
+    if ('dbreader' not in config) and (args.dump or args.sqlbackup):
+        print("Please use --rootconfig to specify which configuration file to use")
+        exit(1)
+
     if args.dump:
         dbmaint.dump(config,args.dump)
         sys.exit()
+
+    if args.sqlbackup:
+        dbmaint.sqlbackup(config,args.sqlbackup,all_databases=True)
+        sys.exit()
+
+

deploy/app/dbmaint.py

@@ -340,6 +340,7 @@
             assert cv == current_version()
 
 def dump(config,dumpdir):
+    """Dump all objects as JSON files and movie files to new directory called DUMP."""
     if os.path.exists(dumpdir):
         raise FileExistsError(f"{dumpdir} exists")
     os.mkdir(dumpdir)
@@ -358,3 +359,22 @@
             json.dump(movie, f, default=str)
         with open(os.path.join(dumpdir,f"movie_{movie_id}.mp4"),"wb") as f:
             f.write(movie_data)
+
+def sqlbackup(config,fname,all_databases=False):
+    """Backup to an sqlfile"""
+    if os.path.exists(fname):
+        raise FileExistsError(f"{fname} exists")
+    dbreader = dbfile.DBMySQLAuth.FromConfig(config['dbreader'])
+    cmd = ['mysqldump','-h' + dbreader.host,'-u' + dbreader.user, '-p' + dbreader.password, '--single-transaction', '--no-tablespaces']
+    if all_databases:
+        cmd.append('--all-databases')
+    else:
+        cmd.append(dbreader.database)
+    if fname.startswith('s3://'):
+        print("Dumping to ",fname)
+        with subprocess.Popen(['aws','s3','cp','-',fname], stdin=subprocess.PIPE) as p_s3:
+            subprocess.run(cmd, stdout=p_s3.stdin)
+        print("done")
+        return
+    with open(fname,'w') as outfile:
+        subprocess.call(cmd, stdout=outfile)

tests/dbreader_test.py

@@ -1,13 +1,16 @@
 import sys
 import os
 import logging
+import uuid
 
+import pytest
 import pymysql
 
+import app.dbmaint as dbmaint
 import app.paths as paths
 import app.dbfile as dbfile
 import app.db as db
-from app.auth import credentials_file,get_dbreader,get_dbwriter,smtp_config
+from app.auth import credentials_file,config,get_dbreader,get_dbwriter,smtp_config
 
 if sys.version < '3.11':
     raise RuntimeError("Requires python 3.11 or above.")
@@ -41,3 +44,12 @@ def test_db_connection():
         raise
     logging.info("MySQL Version %s",v[0][0])
     logging.info("dbwriter: %s",dbwriter)
+
+
+    # Test the dumping
+    dumpdir = '/tmp/dump' + str(uuid.uuid4())
+    dbmaint.dump(config(), dumpdir)
+    dbmaint.sqlbackup(config(), dumpdir + "/sqlbackup.sql", all_databases=True)
+
+    with pytest.raises(FileExistsError):
+        dbmaint.dump(config(), dumpdir)