Major refactoring for including hash generation

Author: fabian-nunes

acquisitionGUI.py

@@ -103,7 +103,7 @@ def extract_thread():
         sg.Push(),
         sg.Text('ADB Extractor', size=(30, 1), justification='center', font=("Helvetica", 25)),
         sg.Push(),
-        sg.Button('?', size=(1, 1), key='-INFO-'),
+        #sg.Button('?', size=(1, 1), key='-INFO-'),
     ],
     [sg.VerticalSeparator(pad=((0, 0), (10, 10)))],
     *options_layout,
@@ -114,16 +114,19 @@ def extract_thread():
     ],
     [sg.VerticalSeparator(pad=((0, 0), (10, 10)))],
     [
+        sg.Push(),
         sg.Column(packages_layout, size=(350, 300)),
         sg.VSeperator(),
         sg.Column(output_layout, size=(400, 300)),
+        sg.Push()
     ],
     [sg.VerticalSeparator(pad=((0, 0), (10, 10)))],
     [
         sg.Push(),
         sg.Button('Extract', size=(15, 1), key='-EXTRACT-'),
         sg.Button('Exit', size=(15, 1), key='-EXIT-'),
-        sg.Text('LabCIF - 2023', size=(30, 1), justification='right', font=("Helvetica", 10)),
+        sg.Push(),
+        #sg.Text('LabCIF - 2023', size=(30, 1), justification='right', font=("Helvetica", 10)),
     ]
 ]
 

modules/adb_acquistion.py

@@ -2,6 +2,7 @@
 import subprocess
 import time
 from datetime import datetime
+import modules.sha_hashes as sha_hashes
 
 
 class Bcolors:
@@ -109,21 +110,28 @@ def get_acquistion(APP, DEVICE, DATA, callback=None, folder=''):
     FILENAME = APP + "-v" + str(VERSION) + "-" + DATA + "--" + DEVNAME + str(ANDROID) + "-u" + str(
         USER) + "--" + datetime.now().strftime("%Y%m%d-%H%M%S") + ".tar"
 
+    OUTPUT_FOLDER = FILENAME[:-4]
+    if folder != '':
+        OUTPUT_FOLDER = folder + "/" + OUTPUT_FOLDER
+    OUTPUT_FILE = OUTPUT_FOLDER + "/" + "sha256_hashes.txt"
+    # create the folder
+    if not os.path.exists(OUTPUT_FOLDER):
+        os.makedirs(OUTPUT_FOLDER)
+
     print_message(callback, "[Info ] " + APP + " version: " + str(VERSION))
     print_message(callback, "[Info ] Android version: " + str(ANDROID))
 
     print_message(callback, "[Info ] Copying data from " + APP + " version " + VERSION + " ...")
 
     if DATA == "private" or DATA == "public":
         if DATA == "private":
-            print("[Info ] Calculating hashes...")
-            hashes_output = subprocess.run(ADB + " " + DEVICE + " shell " + CMD + "'find /data/data/" + APP + END + " -type f -exec sha256sum {} \;'", stdout=subprocess.PIPE, shell=True)
-            output = hashes_output.stdout.decode("utf-8").strip()
             print_message(callback, "[Info ] Acquiring private data")
             # Primary method used to copy the data from the application
             # In windows this process was better when dealing with threads such as in the GUI
             if callback is not None and os.name == 'nt':
-                subprocess.run(ADB + " " + DEVICE + " shell " + CMD + " tar -cvzf /sdcard/Download/" + FILENAME + " /data/data/" + APP + END, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, shell=True)
+                subprocess.run(
+                    ADB + " " + DEVICE + " shell " + CMD + " tar -cvzf /sdcard/Download/" + FILENAME + " /data/data/" + APP + END,
+                    stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, shell=True)
             else:
                 # Method used in the bash script to copy the data from the application
                 # Check for filename with spaces
@@ -138,23 +146,27 @@ def get_acquistion(APP, DEVICE, DATA, callback=None, folder=''):
                     stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, shell=True)
 
         elif DATA == "public":
-            print("[Info ] Calculating hashes...")
-            hashes_output = subprocess.run(
-                ADB + " " + DEVICE + " shell " + CMD + "'find /storage/emulated/0/Android/data/" + APP + END + " -type f -exec sha256sum {} \;'",
-                stdout=subprocess.PIPE, shell=True)
-            output = hashes_output.stdout.decode("utf-8").strip()
             print_message(callback, "[Info ] Acquiring public data")
             # Primary method used to copy the data from the application
             # In windows this process was better when dealing with threads such as in the GUI
             if callback is not None and os.name == 'nt':
-                subprocess.run(ADB + " " + DEVICE + " shell " + CMD + " tar -cvzf /sdcard/Download/" + FILENAME + " /storage/emulated/0/Android/data/" + APP + END, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, shell=True)
+                subprocess.run(
+                    ADB + " " + DEVICE + " shell " + CMD + " tar -cvzf /sdcard/Download/" + FILENAME + " /storage/emulated/0/Android/data/" + APP + END,
+                    stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, shell=True)
             else:
                 # Method used in the bash script to copy the data from the application
                 # Check for filename with spaces
                 subprocess.run(
                     ADB + " " + DEVICE + " shell " + CMD + " tar -cvzf /sdcard/Download/" + FILENAME + " /storage/emulated/0/Android/data/" + APP + END,
                     stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, shell=SHELL)
 
+        OUTPUT_FOLDER = FILENAME[:-4]
+        if folder != '':
+            OUTPUT_FOLDER = folder + "/" + OUTPUT_FOLDER
+        OUTPUT_FILE = OUTPUT_FOLDER + "/" + "sha256_hashes.txt"
+        # create the folder
+        if not os.path.exists(OUTPUT_FOLDER):
+            os.makedirs(OUTPUT_FOLDER)
         # Retrieve the file from the device and save it to the current directory and remove the file from the device
         print_message(callback, "[Info ] Copying to local storage ...")
         print_message(callback, "[Info ] Compressing " + FILENAME + " ...")
@@ -163,25 +175,25 @@ def get_acquistion(APP, DEVICE, DATA, callback=None, folder=''):
         print_message(callback, "[Info] Compressing Terminated.")
 
         print_message(callback, "[Info ] Copying to local storage ...")
-        if folder == '':
-            subprocess.run(ADB + " " + DEVICE + " pull /sdcard/Download/" + FILENAME + ".gz", stdout=subprocess.DEVNULL,
-                           shell=True)
-        else:
-            subprocess.run(ADB + " " + DEVICE + " pull /sdcard/Download/" + FILENAME + ".gz " + folder,
-                           stdout=subprocess.DEVNULL, shell=True)
+        subprocess.run(ADB + " " + DEVICE + " pull /sdcard/Download/" + FILENAME + ".gz " + OUTPUT_FOLDER,
+                       stdout=subprocess.DEVNULL,
+                       shell=True)
         print_message(callback, "[Info ] Copy Terminated.")
         print("[Info ] Creating hashes file...")
         # Check if the command executed successfully
-        if output == '':
-            print_message(callback, "[Info ] No hashes to store.", "error")
-            return
         # Save the output to a file
-        #remove any \r\n
-        H_APP = APP.replace('\r', '')
-        H_FILENAME = H_APP + "-" + DATA + "--sha256_hashes"
-        OUTPUT_FILE = folder + "/" + H_FILENAME + ".txt"
-        with open(OUTPUT_FILE, 'w') as f:
-            f.write(output)
+
+        # unzip the file
+        print_message(callback, "[Info ] Unzipping file...")
+        subprocess.run("gzip -d " + OUTPUT_FOLDER + "/" + FILENAME + ".gz", stdout=subprocess.DEVNULL, shell=True)
+        print_message(callback, "[Info ] Unzip Terminated.")
+        # untar the file
+        print_message(callback, "[Info ] Untaring file...")
+        subprocess.run("tar -xvf " + OUTPUT_FOLDER + "/" + FILENAME + " -C " + OUTPUT_FOLDER, stdout=subprocess.DEVNULL,
+                       stderr=subprocess.DEVNULL, shell=True)
+        print_message(callback, "[Info ] Untar Terminated.")
+        print_message(callback, "[Info ] Calculating hashes.")
+        sha_hashes.calculate_sha256_for_directory(OUTPUT_FOLDER, OUTPUT_FILE)
 
         print_message(callback, "[Info ] Cleaning acquisition files from phone...")
         subprocess.run(ADB + " " + DEVICE + " shell rm /sdcard/Download/" + FILENAME + ".gz", stdout=subprocess.DEVNULL,
@@ -195,31 +207,20 @@ def get_acquistion(APP, DEVICE, DATA, callback=None, folder=''):
         APK = APK.stdout.decode("utf-8").strip()
         APK = APK.split(":")[1]
         print_message(callback, "[Info ] APK: " + APK)
-        if folder == '':
-            subprocess.run(ADB + " " + DEVICE + " pull " + APK + " " + APP + ".apk", shell=True)
-        else:
-            subprocess.run(ADB + " " + DEVICE + " pull " + APK + " " + folder + "/" + APP + ".apk", shell=True)
 
-        #calculate sha256 of the apk
+        subprocess.run(ADB + " " + DEVICE + " pull " + APK + " " + APP + ".apk", stdout=subprocess.DEVNULL, shell=True)
+        os.rename("base.apk", APP + ".apk")
+        # move the apk to the output folder
+        os.rename(APP + ".apk", OUTPUT_FOLDER + "/" + APP + ".apk")
+
+        # calculate sha256 of the apk
         print_message(callback, "[Info ] Calculating hashes...")
-        hashes_output = subprocess.run(
-            ADB + " " + DEVICE + " shell " + CMD + "'sha256sum " + APK + "'",
-            stdout=subprocess.PIPE, shell=True)
-        output = hashes_output.stdout.decode("utf-8").strip()
-        print_message(callback, "[Info ] Creating hashes file...")
-        # Check if the command executed successfully
-        if output == '':
-            print_message(callback, "[Info ] No hashes to store.", "error")
-            return
-        # Save the output to a file
-        # remove any \r\n
-        H_APP = APP.replace('\r', '')
-        H_FILENAME = H_APP + "-" + DATA + "--sha256_hashes"
-        OUTPUT_FILE = folder + "/" + H_FILENAME + ".txt"
-        with open(OUTPUT_FILE, 'w') as f:
-            f.write(output)
-        print_message(callback, "[Done ] Operation Completed with success, generated file: " + APP + ".apk", "ok")
+        sha256 = sha_hashes.calculate_sha256(OUTPUT_FOLDER + "/" + APP + ".apk")
+        # save the sha256 to a file
+        with open(OUTPUT_FILE, "w") as file:
+            file.write(f"{sha256} *{APP}.apk\n")
 
+        print_message(callback, "[Done ] Operation Completed with success, generated file: " + APP + ".apk", "ok")
     else:
-        print_message(callback,  "[ERROR] Invalid data type!", "error")
+        print_message(callback, "[ERROR] Invalid data type!", "error")
         return

modules/sha_hashes.py

@@ -0,0 +1,22 @@
+import hashlib
+import os
+
+
+def calculate_sha256(file_path):
+    sha256_hash = hashlib.sha256()
+    with open(file_path, "rb") as file:
+        for chunk in iter(lambda: file.read(4096), b""):
+            sha256_hash.update(chunk)
+    return sha256_hash.hexdigest()
+
+
+def calculate_sha256_for_directory(root_dir, output_file):
+    with open(output_file, "w") as file:
+        for foldername, subfolders, filenames in os.walk(root_dir):
+            for filename in filenames:
+                file_path = os.path.join(foldername, filename)
+                sha256 = calculate_sha256(file_path)
+                relative_path = os.path.relpath(file_path, root_dir)
+                #ignore if the file is the sha256 file itself
+                if "sha256_hashes.txt" not in relative_path:
+                    file.write(f"{sha256} *{relative_path}\n")