dump MySQL to AWS S3

dbutil.py

@@ -17,7 +17,7 @@
 
 if __name__ == "__main__":
     import argparse
-    parser = argparse.ArgumentParser(description="Database Maintenance Program. The database to act upon is specified in the ini file specified by the PLANTTRACER_CREDENTIALS environment variable, in the sections for [dbreader] and [dbwriter]",
+    parser = argparse.ArgumentParser(description="Database Maintenance Program. The database to act upon is specified in the ini file with the --readconfig or --rootconfig",
                                      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
     required = parser.add_argument_group('required arguments')
     required.add_argument(
@@ -48,7 +48,8 @@
     parser.add_argument("--freshen",help="Non-destructive cleans up the movie metadata for all movies.",action='store_true')
     parser.add_argument("--clean",help="Destructive cleans up the movie metadata for all movies.",action='store_true')
     parser.add_argument("--schema", help="Specify schema file to use", default=paths.SCHEMA_FILE)
-    parser.add_argument("--dump", help="Backup all objects as JSON files and movie files to new directory called DUMP.  ")
+    parser.add_argument("--dump", help="Dump all objects as JSON files and movie files to new directory called DUMP.")
+    parser.add_argument("--sqlbackup", help="Backup the MySQL database to a single SQL file")
     parser.add_argument("--add_admin", help="Add --admin_email user as a course admin to the course specified by --course_id, --course_name, or --course_name", action='store_true')
     parser.add_argument("--course_id", help="integer course id", type=int)
     parser.add_argument("--course_key", help="integer course id")
@@ -236,6 +237,17 @@
         dbmaint.freshen(True)
         sys.exit(0)
 
+    # These need a database config
+    if ('dbreader' not in config) and (args.dump or args.sqlbackup):
+        print("Please use --rootconfig to specify which configuration file to use")
+        exit(1)
+
     if args.dump:
         dbmaint.dump(config,args.dump)
         sys.exit()
+
+    if args.sqlbackup:
+        dbmaint.sqlbackup(config,args.sqlbackup,all_databases=True)
+        sys.exit()
+
+

deploy/app/dbmaint.py

@@ -340,6 +340,7 @@
             assert cv == current_version()
 
 def dump(config,dumpdir):
+    """Dump all objects as JSON files and movie files to new directory called DUMP."""
     if os.path.exists(dumpdir):
         raise FileExistsError(f"{dumpdir} exists")
     os.mkdir(dumpdir)
@@ -358,3 +359,22 @@
             json.dump(movie, f, default=str)
         with open(os.path.join(dumpdir,f"movie_{movie_id}.mp4"),"wb") as f:
             f.write(movie_data)
+
+def sqlbackup(config,fname,all_databases=False):
+    """Backup to an sqlfile"""
+    if os.path.exists(fname):
+        raise FileExistsError(f"{fname} exists")
+    dbreader = dbfile.DBMySQLAuth.FromConfig(config['dbreader'])
+    cmd = ['mysqldump','-h' + dbreader.host,'-u' + dbreader.user, '-p' + dbreader.password, '--single-transaction', '--no-tablespaces']
+    if all_databases:
+        cmd.append('--all-databases')
+    else:
+        cmd.append(dbreader.database)
+    if fname.startswith('s3://'):
+        print("Dumping to ",fname)
+        with subprocess.Popen(['aws','s3','cp','-',fname], stdin=subprocess.PIPE) as p_s3:
+            subprocess.run(cmd, stdout=p_s3.stdin)
+        print("done")
+        return
+    with open(fname,'w') as outfile:
+        subprocess.call(cmd, stdout=outfile)