small improvements with compression management

Author: sagehen03

batch-compression-management/bgz.yml

@@ -54,7 +54,7 @@ Resources:
       PlatformCapabilities:
         - 'FARGATE'
       Timeout:
-        AttemptDurationSeconds: 7200
+        AttemptDurationSeconds: 20400
       ContainerProperties:
         Image: !Sub '${AWS::AccountId}.dkr.ecr.us-east-1.amazonaws.com/bgzip-repo'
         Command:
@@ -86,7 +86,7 @@ Resources:
       PlatformCapabilities:
         - 'FARGATE'
       Timeout:
-        AttemptDurationSeconds: 7200
+        AttemptDurationSeconds: 20400
       ContainerProperties:
         Image: !Sub '${AWS::AccountId}.dkr.ecr.us-east-1.amazonaws.com/bgzip-repo'
         Command:

batch-compression-management/compress_json_files.py

@@ -40,7 +40,7 @@ def bg_compress_and_index_file(bucket_name, file, boto_s3, files_to_retry, print
     # Run bgzip compression and capture stderr
     command = ['bgzip', '-i', f"s3://{bucket_name}/{file}"]
     try:
-        result = subprocess.run(command, check=True, timeout=600,
+        result = subprocess.run(command, check=True, timeout=1200,
                                stdout=subprocess.DEVNULL, stderr=subprocess.PIPE,
                                universal_newlines=True)
         stderr_output = result.stderr

batch-compression-management/delete_json_files.py

@@ -19,6 +19,7 @@ def main(index, bucket, path):
 
 
 def delete_json_file(bucket_name, file, boto_s3, files_to_retry, print_lock):
+    print("Attempting to delete file: ", file)
     results = boto_s3.list_objects(Bucket=bucket_name, Prefix=file + ".gz")
     if results.get('Contents', None) and len(results.get('Contents')) == 2:
         with print_lock: