Adding the rollup logic to the archive. Fixing a typo in the rollup script (aFile->oFile).

Temporary file now removed after good write to weekly.

Author: John Dunham

csmdb/sql/csm_db_history_archive.py

@@ -36,6 +36,7 @@
 from datetime import date
 import threading
 from multiprocessing.dummy import Pool as ThreadPool
+from csm_db_rollup import  rollupDir
 
 DEFAULT_LOG='''/var/log/ibm/csm/db/csm_db_archive_script.log'''
 DEFAULT_TARGET='''/var/log/ibm/csm/archive'''
@@ -225,6 +226,13 @@ def main(args):
     # Verifies path exists.
     if not os.path.exists(args.target):
         os.makedirs(args.target)
+
+    # Make the temp directory for staging the tables for dumping.
+    # This is used before writing to mitigate potential data loss.
+    temp_dir = "{0}/tmp".format(args.target)
+    if not os.path.exists(temp_dir):
+        os.makedirs(temp_dir)
+ 
 
     # Process the script detail info. for screen and logging.
     logging.info("DB Name:                           | {0}".format(args.db))
@@ -243,18 +251,21 @@ def main(args):
 
     pool = ThreadPool(int(args.threads))
 
-    tmp_list =  pool.map( lambda table: dump_table( args.db, args.user, table, args.count, args.target ), TABLES )
+    tmp_list =  pool.map( lambda table: dump_table( args.db, args.user, table, args.count, temp_dir ), TABLES )
 
     for entry in tmp_list:
         if entry is None:0
         else:
             print entry
 
     for table in RAS_TABLES:
-        entry = dump_table( args.db, args.user, table, args.count, args.target, True)
+        entry = dump_table( args.db, args.user, table, args.count, temp_dir, True)
         if entry is None:0
         else:
-            print entry
+            print entry args.target
+
+    # After the tables are dumped, it's time to merge them into the weekly report.
+    rollupDir(temp_dir,"..")
 
     # Process the finishing info. for screen and logging.
     ft = datetime.datetime.now()

csmdb/sql/csm_db_rollup.py

@@ -27,19 +27,20 @@
 
 DEFAULT_TARGET='''/var/log/ibm/csm/archive'''
 
-def rollupDir (directory):
+def rollupDir (directory, archiveTarget="old"):
     ''' Performs a rollup operation on a directory, designed to condense in to weeks of data. 
     This requires the files provided are in the format `<table>.archive.<date>.json`
 
     directory -- A directory containing a collection of archive files to condense (string).
+    archiveTarget -- A directory to store the rollups in.
     '''
     # If the directory is not present return early.
     if not os.path.exists(directory):
         print("{0} does not exist".format(directory))
         return
 
-    # Create an srchhive directory as needed. 
-    archiveDir="{0}/{1}".format(directory,"old")
+    # Create an archive directory as needed. 
+    archiveDir="{0}/{1}".format(directory,archiveTarget)
     if not os.path.exists(archiveDir): 
         os.makedirs(archiveDir)
 
@@ -56,24 +57,23 @@ def rollupDir (directory):
         try:
             sFile=f.split(".")
             weekStr=datetime.strptime(sFile[2],"%Y-%m-%d").strftime("%Y-%U")
-            aFile="{0}/{1}-{2}.json".format(archiveDir, sFile[0], weekStr)
+            oFile="{0}/{1}-{2}.json".format(archiveDir, sFile[0], weekStr)
         except:
             continue
 
         # Get the contents first
         contents=""
         with open(iFile,'r') as inputFile:
             contents=inputFile.read()
-
             # remove and skip empties. 
             if len(contents) == 0:
-                os.remove(iFile)
+                os.remove(iFile) 
                 continue
 
         # Archive the contents.
         with open(oFile, 'a') as ofile:
             ofile.write(contents)
-            os.remove(oFile)
+            os.remove(iFile) # Remove only if the write was good!
 
 def main(args):
     target = DEFAULT_TARGET