Added backend handling of overwriting on import pre-cracked hashes and import by file and URL

ci/apiv2/hashtopolis.py

@@ -1023,11 +1023,13 @@ def export_wordlist(self, hashlist):
         response = self._helper_request("exportWordlist", payload)
         return File(**response['data'])
 
-    def import_cracked_hashes(self, hashlist, source_data: str, separator):
+    def import_cracked_hashes(self, hashlist, source_type, source_data: str, separator, overwrite):
         payload = {
             'hashlistId': hashlist.id,
+            'sourceType': source_type,
             'sourceData': base64.b64encode(source_data.encode()).decode(),
             'separator': separator,
+            'overwrite': overwrite,
         }
         response = self._helper_request("importCrackedHashes", payload)
         return response['meta']

ci/apiv2/test_hashlist.py

@@ -68,7 +68,7 @@ def test_export_wordlist(self):
         cracked = "cc03e747a6afbbcbf8be7668acfebee5:test123"
 
         helper = Helper()
-        helper.import_cracked_hashes(model_obj, cracked, ':')
+        helper.import_cracked_hashes(model_obj, 'paste', cracked, ':', 0)
 
         file = helper.export_wordlist(model_obj)
 
@@ -84,7 +84,7 @@ def test_import_cracked_hashes(self):
         cracked = "cc03e747a6afbbcbf8be7668acfebee5:test123"
 
         helper = Helper()
-        result = helper.import_cracked_hashes(model_obj, cracked, ':')
+        result = helper.import_cracked_hashes(model_obj, 'paste', cracked, ':', 0)
 
         self.assertEqual(result['totalLines'], 1)
         self.assertEqual(result['newCracked'], 1)
@@ -98,7 +98,7 @@ def test_import_cracked_hashes_invalid(self):
         cracked = "cc03e747a6afbbcbf8be7668acfebee5__test123"
 
         helper = Helper()
-        result = helper.import_cracked_hashes(model_obj, cracked, ':')
+        result = helper.import_cracked_hashes(model_obj, 'paste', cracked, ':', 0)
 
         self.assertEqual(result['totalLines'], 1)
         self.assertEqual(result['invalid'], 1)
@@ -112,7 +112,7 @@ def test_import_cracked_hashes_notfound(self):
         cracked = "ffffffffffffffffffffffffffffffff:test123"
 
         helper = Helper()
-        result = helper.import_cracked_hashes(model_obj, cracked, ':')
+        result = helper.import_cracked_hashes(model_obj, 'paste', cracked, ':', 0)
 
         self.assertEqual(result['totalLines'], 1)
         self.assertEqual(result['notFound'], 1)
@@ -126,9 +126,9 @@ def test_import_cracked_hashes_already_cracked(self):
         cracked = "cc03e747a6afbbcbf8be7668acfebee5:test123"
 
         helper = Helper()
-        helper.import_cracked_hashes(model_obj, cracked, ':')
+        helper.import_cracked_hashes(model_obj, 'paste', cracked, ':', 0)
 
-        result = helper.import_cracked_hashes(model_obj, cracked, ':')
+        result = helper.import_cracked_hashes(model_obj, 'paste', cracked, ':', 0)
 
         self.assertEqual(result['totalLines'], 1)
         self.assertEqual(result['alreadyCracked'], 1)

ci/tests/HashlistTest.class.php

@@ -109,6 +109,7 @@ private function testImportCracked() {
       "request" => "importCracked",
       "hashlistId" => 1,
       "separator" => ":",
+      "overwrite" => 0,
       // sending 3 founds of the hashlist
       "data" => "MDAyODA4MGU3ZmE4YzgxMjY4ZWYzNDBkN2Q2OTI2ODE6Zm91bmQxCjAwMmU5NWQ4MmJlMzAzOTZmY2NkMzc1ZmYyM2Y4YjRjOmZvdW5kMgowMDM0YzVlNDE4YWU0ZjJlYmE1OTBhMTY2OTZlZGJiMzpmb3VuZDM=",
       "accessKey" => "mykey"

src/inc/apiv2/helper/importCrackedHashes.routes.php

@@ -26,8 +26,10 @@ public function getRequiredPermissions(string $method): array {
   public function getFormFields(): array {
     return [
       Hashlist::HASHLIST_ID => ["type" => "int"],
+      "sourceType" => ['type' => 'str'],
       "sourceData" => ['type' => 'str'],
       "separator" => ['type' => 'str'],
+      "overwrite" => ['type' => 'int'],
     ];
   }
 
@@ -46,13 +48,38 @@ public static function getResponse(): array {
   /**
    * Endpoint to import cracked hashes into a hashlist.
    * @throws HTException
+   * @throws HttpError
    */
   public function actionPost($data): object|array|null {
     $hashlist = self::getHashlist($data[Hashlist::HASHLIST_ID]);
 
-    $importData = base64_decode($data["sourceData"]);
+    // Cast to processZap compatible upload format
+    $dummyPost = [];
+    switch ($data["sourceType"]) {
+      case "paste":
+        $dummyPost["hashfield"] = base64_decode($data["sourceData"]);
+        break;
+      case "import":
+        $dummyPost["importfile"] = $data["sourceData"];
+        break;
+      case "url":
+        $dummyPost["url"] = $data["sourceData"];
+        break;
+      default:
+        // TODO: Choice validation are model based checks
+        throw new HttpErrorException("sourceType value '" . $data["sourceType"] . "' is not supported (choices paste, import, url");
+    }
+
+    if ($data["sourceType"] == "paste") {
+      if (strlen($data["sourceData"]) == 0) {
+        throw new HttpError("sourceType=paste, requires sourceData to be non-empty");
+      }
+      else if ($dummyPost["hashfield"] === false) {
+        throw new HttpError("sourceData not valid base64 encoding");
+      }
+    }
 
-    $result = HashlistUtils::processZap($hashlist->getId(), $data["separator"], "paste", ["hashfield" => $importData], [], $this->getCurrentUser());
+    $result = HashlistUtils::processZap($hashlist->getId(), $data["separator"], $data["sourceType"], $dummyPost, [], $this->getCurrentUser(), (isset($data["overwrite"]) && intval($data["overwrite"]) == 1) ? true : false);
 
     return [
       "totalLines" => $result[0],

src/inc/apiv2/model/hashlists.routes.php

@@ -102,6 +102,7 @@ public function getFormFields(): array {
 
   /**
    * @throws HttpErrorException
+   * @throws HttpError
    * @throws HTException
    */
   protected function createObject(array $data): int {
@@ -122,11 +123,12 @@ protected function createObject(array $data): int {
         throw new HttpErrorException("sourceType value '" . $data["sourceType"] . "' is not supported (choices paste, import, url");
     }
 
-    // TODO: validate input is valid base64 encoded
     if ($data["sourceType"] == "paste") {
       if (strlen($data["sourceData"]) == 0) {
-        // TODO: Should be 400 instead
-        throw new HttpErrorException("sourceType=paste, requires sourceData to be non-empty");
+        throw new HttpError("sourceType=paste, requires sourceData to be non-empty");
+      }
+      else if ($dummyPost["hashfield"] === false) {
+        throw new HttpError("sourceData not valid base64 encoding");
       }
     }
 

src/inc/handlers/HashlistHandler.class.php

@@ -49,7 +49,7 @@ public function handle($action) {
           break;
         case DHashlistAction::PROCESS_ZAP:
           AccessControl::getInstance()->checkPermission(DHashlistAction::PROCESS_ZAP_PERM);
-          $data = HashlistUtils::processZap($_POST['hashlist'], $_POST['separator'], $_POST['source'], $_POST, $_FILES, AccessControl::getInstance()->getUser());
+          $data = HashlistUtils::processZap($_POST['hashlist'], $_POST['separator'], $_POST['source'], $_POST, $_FILES, AccessControl::getInstance()->getUser(), (isset($_POST["overwrite"]) && intval($_POST["overwrite"]) == 1) ? true : false);
           UI::addMessage(UI::SUCCESS, "Processed pre-cracked hashes: " . $data[0] . " total lines, " . $data[1] . " new cracked hashes, " . $data[2] . " were already cracked, " . $data[3] . " invalid lines, " . $data[4] . " not matching entries (" . $data[5] . "s)!");
           if ($data[6] > 0) {
             UI::addMessage(UI::WARN, $data[6] . " entries with too long plaintext");

src/inc/user-api/UserAPIHashlist.class.php

@@ -182,7 +182,8 @@ private function importCracked($QUERY) {
       'paste',
       ['hashfield' => base64_decode($QUERY[UQueryHashlist::HASHLIST_DATA])],
       [],
-      $this->user
+      $this->user,
+      false
     );
     $response = [
       UResponseHashlist::SECTION => $QUERY[UQueryHashlist::SECTION],

src/inc/utils/HashlistUtils.class.php

@@ -305,10 +305,11 @@ public static function rename($hashlistId, $name, $user) {
    * @param array $post
    * @param array $files
    * @param User $user
+   * @param boolean $overwritePlaintext
    * @return int[]
    * @throws HTException
    */
-  public static function processZap($hashlistId, $separator, $source, $post, $files, $user) {
+  public static function processZap($hashlistId, $separator, $source, $post, $files, $user, $overwritePlaintext) {
     // pre-crack hashes processor
     $hashlist = HashlistUtils::getHashlist($hashlistId);
     if (!AccessUtils::userCanAccessHashlists($hashlist, $user)) {
@@ -429,16 +430,23 @@ public static function processZap($hashlistId, $separator, $source, $post, $file
         }
         else if ($hashEntry->getIsCracked() == 1) {
           $alreadyCracked++;
-          continue;
+          if (!$overwritePlaintext) {
+            continue;
+          }
         }
         $plain = str_replace($hash . $separator . $hashEntry->getSalt() . $separator, "", $data);
         if (strlen($plain) > SConfig::getInstance()->getVal(DConfig::PLAINTEXT_MAX_LENGTH)) {
           $tooLong++;
           continue;
         }
+
+        if ($hashEntry->getIsCracked() != 1) {
+          $newCracked++;
+          $crackedIn[$hashEntry->getHashlistId()]++;
+        }
+
         $hashFactory->mset($hashEntry, [Hash::PLAINTEXT => $plain, Hash::IS_CRACKED => 1, Hash::TIME_CRACKED => time()]);
-        $newCracked++;
-        $crackedIn[$hashEntry->getHashlistId()]++;
+
         if ($hashlist->getFormat() == DHashlistFormat::PLAIN) {
           $zaps[] = new Zap(null, $hashEntry->getHash(), time(), null, $hashlist->getId());
         }
@@ -471,19 +479,28 @@ public static function processZap($hashlistId, $separator, $source, $post, $file
         foreach ($hashEntries as $hashEntry) {
           if ($hashEntry->getIsCracked() == 1) {
             $alreadyCracked++;
-            continue;
+            if (!$overwritePlaintext) {
+              continue;
+            }
           }
+
           $plain = str_replace($hash . $separator, "", $data);
+
           if (strlen($plain) > SConfig::getInstance()->getVal(DConfig::PLAINTEXT_MAX_LENGTH)) {
             $tooLong++;
             continue;
           }
+
+          if ($hashEntry->getIsCracked() != 1) {
+            $newCracked++;
+            $crackedIn[$hashEntry->getHashlistId()]++;
+          }
+
           $hashFactory->mset($hashEntry, [Hash::PLAINTEXT => $plain, Hash::IS_CRACKED => 1, Hash::TIME_CRACKED => time()]);
-          $crackedIn[$hashEntry->getHashlistId()]++;
+
           if ($hashlist->getFormat() == DHashlistFormat::PLAIN) {
             $zaps[] = new Zap(null, $hashEntry->getHash(), time(), null, $hashlist->getId());
           }
-          $newCracked++;
         }
       }
       $bufferCount++;