Implemented a better handling of (partially) unsalted hashlists #1657

ci/tests/integration/MaxAgentsTest.class.php

@@ -49,7 +49,7 @@ public function run() {
     HashtopolisTestFramework::log(HashtopolisTestFramework::LOG_INFO, "Running " . $this->getTestName() . "...");
     $this->prepare();
     try {
-      $response = $this->addHashlist(["name" => "NotSecureList", "isSecure" => false]);
+      $response = $this->addHashlist(["name" => "NotSecureList", "isSecure" => false])["hashlist"];
       $hashlistId = $response["hashlistId"];
 
       $this->testTaskMaxAgents($hashlistId);

ci/tests/integration/RuleSplitTest.class.php

@@ -59,7 +59,7 @@ private function testRuleSplit() {
     $file_id2 = $this->getFile('best64.rule');
 
     # Create hashlist
-    $response = $this->addHashlist(["isSecure" => false]);
+    $response = $this->addHashlist(["isSecure" => false])["hashlist"];
     $hashlistId = $response["hashlistId"];
 
     # Create task with rule/wordlist

src/inc/apiv2/common/AbstractBaseAPI.class.php

@@ -1485,7 +1485,7 @@ static function createJsonResponse(array $data = [], array $links = [], array $i
   /**
    * Get single Resource
    */
-  protected static function getOneResource(object $apiClass, object $object, Request $request, Response $response, int $statusCode = 200): Response {
+  protected static function getOneResource(object $apiClass, object $object, Request $request, Response $response, int $statusCode = 200, array $creationInformation = null): Response {
     $apiClass->preCommon($request);
 
     $validExpandables = $apiClass->getExpandables();
@@ -1524,6 +1524,10 @@ protected static function getOneResource(object $apiClass, object $object, Reque
     if ($apiClass->permissionErrors !== null) {
       $metaData["Include errors"] = $apiClass->permissionErrors;
     }
+    if(is_array($creationInformation)) {
+      $metaData["creationInformation"] = $creationInformation;
+    }
+
     // Generate JSON:API GET output
     $ret = self::createJsonResponse($dataResources[0], $links, $includedResources, $metaData);
 

src/inc/apiv2/common/AbstractHelperAPI.class.php

@@ -12,8 +12,8 @@ abstract class AbstractHelperAPI extends AbstractBaseAPI {
   abstract public function actionPost(array $data): object|array|null;
 
   /**
-   * Function in order to create swagger documentation. SHould return either a map of strings that
-   * describes the output ex: ["assign" => "succes"] or if the endpoint returns an object it should return
+   * Function in order to create swagger documentation. Should return either a map of strings that
+   * describes the output ex: ["assign" => "success"] or if the endpoint returns an object it should return
    * the string representation of that object ex: File.
    */
   abstract public static function getResponse(): array|string|null;

src/inc/apiv2/common/AbstractModelAPI.class.php

@@ -20,9 +20,13 @@ abstract class AbstractModelAPI extends AbstractBaseAPI {
   abstract static public function getDBAClass();
 
   abstract protected function createObject(array $data): int;
-  
+
   abstract protected function deleteObject(object $object): void;
 
+  protected function createObjectAndGetResult(array $data): array {
+    return [];
+  }
+
   public static function getToOneRelationships(): array {
     return [];
   }
@@ -1147,11 +1151,21 @@ public function post(Request $request, Response $response, array $args): Respons
 
     // Remove key aliases and sanitize to 'db values and request creation
     $mappedData = $this->unaliasData($attributes, $allFeatures);
-    $pk = $this->createObject($mappedData);
 
-    // Request object again, since post-modified entries are not reflected into object.
-    $object = $this->getFactory()->get($pk);
-    return self::getOneResource($this, $object, $request, $response, 201);
+    $object = null;
+
+    if (isset($data["getCreationInformation"])) {
+      $creationResult = $this->createObjectAndGetResult($mappedData);
+      // Request object again, since post-modified entries are not reflected into object.
+      $object = $this->getFactory()->get($creationResult["pk"]);
+      return self::getOneResource($this, $object, $request, $response, 201, $creationResult["creationInformation"]);
+    }
+    else {
+      $pk = $this->createObject($mappedData);
+      // Request object again, since post-modified entries are not reflected into object.
+      $object = $this->getFactory()->get($pk);
+      return self::getOneResource($this, $object, $request, $response, 201);
+    }
   }
 
 

src/inc/apiv2/model/hashlists.routes.php

@@ -102,9 +102,19 @@ public function getFormFields(): array {
 
   /**
    * @throws HttpErrorException
+   * @throws HttpError
    * @throws HTException
    */
   protected function createObject(array $data): int {
+    return $this->createObjectAndGetResult($data)["pk"];
+  }
+
+  /**
+   * @throws HttpErrorException
+   * @throws HttpError
+   * @throws HTException
+   */
+  protected function createObjectAndGetResult(array $data): array {
     // Cast to createHashlist compatible upload format
     $dummyPost = [];
     switch ($data["sourceType"]) {
@@ -122,15 +132,16 @@ protected function createObject(array $data): int {
         throw new HttpErrorException("sourceType value '" . $data["sourceType"] . "' is not supported (choices paste, import, url");
     }
 
-    // TODO: validate input is valid base64 encoded
     if ($data["sourceType"] == "paste") {
       if (strlen($data["sourceData"]) == 0) {
-        // TODO: Should be 400 instead
-        throw new HttpErrorException("sourceType=paste, requires sourceData to be non-empty");
+        throw new HttpError("sourceType=paste, requires sourceData to be non-empty");
+      }
+      else if ($dummyPost["hashfield"] === false) {
+        throw new HttpError("sourceData not valid base64 encoding");
       }
     }
 
-    $hashlist = HashlistUtils::createHashlist(
+    $hashlistData = HashlistUtils::createHashlist(
       $data[Hashlist::HASHLIST_NAME],
       $data[Hashlist::IS_SALTED],
       $data[Hashlist::IS_SECRET],
@@ -150,11 +161,14 @@ protected function createObject(array $data): int {
 
     // Modify fields not set on hashlist creation
     if (array_key_exists("notes", $data)) {
-      HashlistUtils::editNotes($hashlist->getId(), $data["notes"], $this->getCurrentUser());
-    };
-    HashlistUtils::setArchived($hashlist->getId(), $data[UQueryHashlist::HASHLIST_IS_ARCHIVED], $this->getCurrentUser());
+      HashlistUtils::editNotes($hashlistData["hashlist"]->getId(), $data["notes"], $this->getCurrentUser());
+    }
+    HashlistUtils::setArchived($hashlistData["hashlist"]->getId(), $data[UQueryHashlist::HASHLIST_IS_ARCHIVED], $this->getCurrentUser());
 
-    return $hashlist->getId();
+    $creationResult["pk"] = $hashlistData["hashlist"]->getId();
+    $creationResult["creationInformation"] = $hashlistData["statistics"];
+
+    return $creationResult;
   }
 
   /**
@@ -175,4 +189,4 @@ protected function getUpdateHandlers($id, $current_user): array {
   }
 }
 
-HashlistAPI::register($app);
+HashlistAPI::register($app);

src/inc/handlers/HashlistHandler.class.php

@@ -92,7 +92,7 @@ public function handle($action) {
             AccessControl::getInstance()->getUser(),
             (isset($_POST["useBrain"]) && intval($_POST["useBrain"]) == 1) ? 1 : 0,
             (isset($_POST['brain-features'])) ? intval($_POST['brain-features']) : 0
-          );
+          )["hashlist"];
           header("Location: hashlists.php?id=" . $hashlist->getId());
           die();
         case DHashlistAction::CREATE_SUPERHASHLIST:

src/inc/user-api/UserAPIHashlist.class.php

@@ -273,7 +273,7 @@ private function createHashlist($QUERY) {
       $this->user,
       $QUERY[UQueryHashlist::HASHLIST_USE_BRAIN],
       $QUERY[UQueryHashlist::HASHLIST_BRAIN_FEATURES]
-    );
+    )["hashlist"];
     $this->sendResponse(array(
         UResponseHashlist::SECTION => $QUERY[UQuery::SECTION],
         UResponseHashlist::REQUEST => $QUERY[UQuery::REQUEST],

src/inc/utils/HashlistUtils.class.php

@@ -747,7 +747,8 @@ public static function export($hashlistId, $user) {
    * @param User $user
    * @param int $brainId
    * @param int $brainFeatures
-   * @return Hashlist
+   * @param boolean $writeResultsToNotes
+   * @return array
    * @throws HTException
    */
   public static function createHashlist($name, $isSalted, $isSecret, $isHexSalted, $separator, $format, $hashtype, $saltSeparator, $accessGroupId, $source, $post, $files, $user, $brainId, $brainFeatures) {
@@ -816,33 +817,62 @@ public static function createHashlist($name, $isSalted, $isSecret, $isHexSalted,
       Factory::getAgentFactory()->getDB()->rollback();
       throw new HttpError("Hashlist has too many lines!");
     }
+
     $file = fopen($tmpfile, "rb");
     if (!$file) {
       Factory::getAgentFactory()->getDB()->rollback();
       throw new HttpError("Failed to open file!");
     }
+
+    if ($format == DHashlistFormat::PLAIN && $salted) {
+      // find out if the file contains a salt separator at all
+      rewind($file);
+
+      $saltSeparatorFound = false;
+      while (($currentLine = fgets($file)) !== false) {
+        if (strpos($currentLine, $saltSeparator) !== false) {
+          $saltSeparatorFound = true;
+          break;
+        }
+      }
+
+      if ($saltSeparatorFound === false) {
+        fclose($file);
+        unlink($tmpfile);
+        Factory::getAgentFactory()->getDB()->rollback();
+
+        throw new HttpError("Salted hashes separator not found at all in the hashlist! Hashlist not created.");
+      }
+    }
+    else {
+      $saltSeparator = "";
+    }
+
+    Factory::getAgentFactory()->getDB()->commit();
+
     $added = 0;
     $preFound = 0;
+    $hashlistStatistics = [];
+    $hashlistStatistics["uploadedTotalLines"] = 0;
+    $hashlistStatistics["uploadedEmptyLines"] = 0;
+    $hashlistStatistics["uploadedValidHashes"] = 0;
+    $hashlistStatistics["uploadedValidHashesWithoutExpectedSalt"] = 0;
+    $hashlistStatistics["uploadedInvalidHashes"] = 0;
 
     switch ($format) {
       case DHashlistFormat::PLAIN:
-        if ($salted) {
-          // find out if the first line contains field separator
-          rewind($file);
-          $bufline = stream_get_line($file, 1024);
-          if (strpos($bufline, $saltSeparator) === false) {
-            throw new HttpError("Salted hashes separator not found in file!");
-          }
-        }
-        else {
-          $saltSeparator = "";
-        }
         rewind($file);
+
+        Factory::getAgentFactory()->getDB()->beginTransaction();
         $values = array();
         $bufferCount = 0;
+
         while (!feof($file)) {
           $line = trim(fgets($file));
+          $hashlistStatistics["uploadedTotalLines"]++;
+
           if (strlen($line) == 0) {
+            $hashlistStatistics["uploadedEmptyLines"]++;
             continue;
           }
           $hash = $line;
@@ -853,8 +883,12 @@ public static function createHashlist($name, $isSalted, $isSecret, $isHexSalted,
               $hash = substr($line, 0, $pos);
               $salt = substr($line, $pos + 1);
             }
+            else {
+              $hashlistStatistics["uploadedValidHashesWithoutExpectedSalt"]++;
+            }
           }
           if (strlen($hash) == 0) {
+            $hashlistStatistics["uploadedEmptyLines"]++;
             continue;
           }
           //TODO: check hash length here
@@ -871,42 +905,62 @@ public static function createHashlist($name, $isSalted, $isSecret, $isHexSalted,
               }
             }
           }
+
           if ($found == null) {
             $values[] = new Hash(null, $hashlist->getId(), $hash, $salt, "", 0, null, 0, 0);
           }
           else {
             $values[] = new Hash(null, $hashlist->getId(), $hash, $salt, $found->getPlaintext(), time(), null, 1, 0);
             $preFound++;
           }
+
           $bufferCount++;
+          $hashlistStatistics["uploadedValidHashes"]++;
+
           if ($bufferCount >= 10000) {
             $result = Factory::getHashFactory()->massSave($values);
             $added += $result->rowCount();
             $values = array();
             $bufferCount = 0;
           }
         }
+
         if (sizeof($values) > 0) {
           $result = Factory::getHashFactory()->massSave($values);
           $added += $result->rowCount();
         }
+
         fclose($file);
         unlink($tmpfile);
+
+        if ($added === 0) {
+          Factory::getAgentFactory()->getDB()->rollback();
+          Factory::getHashlistFactory()->delete($hashlist);
+          Factory::getAgentFactory()->getDB()->commit();
+          throw new HttpError("No valid hashes found! Hashlist not created.");
+        }
+
         Factory::getHashlistFactory()->mset($hashlist, [Hashlist::HASH_COUNT => $added, Hashlist::CRACKED => $preFound]);
-        Util::createLogEntry("User", $user->getId(), DLogEntry::INFO, "New Hashlist created: " . $hashlist->getHashlistName());
+        Factory::getAgentFactory()->getDB()->commit();
+        Util::createLogEntry("User", $user->getId(), DLogEntry::INFO, "New Hashlist created: " . $hashlist->getHashlistName() . ". Total lines: " . $hashlistStatistics["uploadedTotalLines"] . " Empty lines: " . $hashlistStatistics["uploadedEmptyLines"] . " Valid hashes: " . $hashlistStatistics["uploadedValidHashes"] . " Valid hashes without expected salt: " . $hashlistStatistics["uploadedValidHashesWithoutExpectedSalt"] . " Invalid hashes: " . $hashlistStatistics["uploadedInvalidHashes"]);
 
         NotificationHandler::checkNotifications(DNotificationType::NEW_HASHLIST, new DataSet(array(DPayloadKeys::HASHLIST => $hashlist)));
         break;
       case DHashlistFormat::WPA:
         $added = 0;
         $values = [];
+
         while (!feof($file)) {
+          $hashlistStatistics["uploadedTotalLines"]++;
+
           if ($hashlist->getHashTypeId() == 2500) { // HCCAPX hashes
             $data = fread($file, 393);
             if (strlen($data) == 0) {
+              $hashlistStatistics["uploadedInvalidHashes"]++;
               break;
             }
             if (strlen($data) != 393) {
+              $hashlistStatistics["uploadedInvalidHashes"]++;
               UI::printError("ERROR", "Data file only contains " . strlen($data) . " bytes!");
             }
             // get the SSID
@@ -934,11 +988,13 @@ public static function createHashlist($name, $isSalted, $isSecret, $isHexSalted,
             $mac_cli = Util::bintohex($mac_cli);
             $hash = new HashBinary(null, $hashlist->getId(), $mac_ap . SConfig::getInstance()->getVal(DConfig::FIELD_SEPARATOR) . $mac_cli . SConfig::getInstance()->getVal(DConfig::FIELD_SEPARATOR) . Util::bintohex($network), Util::bintohex($data), null, 0, null, 0, 0);
             Factory::getHashBinaryFactory()->save($hash);
+            $hashlistStatistics["uploadedValidHashes"]++;
             $added++;
           }
           else { // PMKID hashes
             $line = trim(fgets($file));
             if (strlen($line) == 0) {
+              $hashlistStatistics["uploadedEmptyLines"]++;
               continue;
             }
             if (strpos($line, "*") !== false) {
@@ -957,14 +1013,15 @@ public static function createHashlist($name, $isSalted, $isSecret, $isHexSalted,
             }
             $hash = new HashBinary(null, $hashlist->getId(), $identification, Util::bintohex($line . "\n"), null, 0, null, 0, 0);
             Factory::getHashBinaryFactory()->save($hash);
+            $hashlistStatistics["uploadedValidHashes"]++;
             $added++;
           }
         }
         fclose($file);
         unlink($tmpfile);
 
         Factory::getHashlistFactory()->set($hashlist, Hashlist::HASH_COUNT, $added);
-        Util::createLogEntry("User", $user->getId(), DLogEntry::INFO, "New Hashlist created: " . $hashlist->getHashlistName());
+        Util::createLogEntry("User", $user->getId(), DLogEntry::INFO, "New Hashlist created: " . $hashlist->getHashlistName() . ". Total lines: " . $hashlistStatistics["uploadedTotalLines"] . " Empty lines: " . $hashlistStatistics["uploadedEmptyLines"] . " Valid hashes: " . $hashlistStatistics["uploadedValidHashes"] . " Valid hashes without expected salt: " . $hashlistStatistics["uploadedValidHashesWithoutExpectedSalt"] . " Invalid hashes: " . $hashlistStatistics["uploadedInvalidHashes"]);
 
         NotificationHandler::checkNotifications(DNotificationType::NEW_HASHLIST, new DataSet(array(DPayloadKeys::HASHLIST => $hashlist)));
         break;
@@ -973,17 +1030,21 @@ public static function createHashlist($name, $isSalted, $isSecret, $isHexSalted,
           $data = fread($file, Util::filesize($tmpfile));
           $hash = new HashBinary(null, $hashlist->getId(), "", Util::bintohex($data), "", 0, null, 0, 0);
           Factory::getHashBinaryFactory()->save($hash);
+          $hashlistStatistics["uploadedValidHashes"]++;
         }
+
         fclose($file);
         unlink($tmpfile);
+
         Factory::getHashlistFactory()->set($hashlist, Hashlist::HASH_COUNT, 1);
-        Util::createLogEntry("User", $user->getId(), DLogEntry::INFO, "New Hashlist created: " . $hashlist->getHashlistName());
+        Util::createLogEntry("User", $user->getId(), DLogEntry::INFO, "New Hashlist created: " . $hashlist->getHashlistName() . ". Total lines: " . $hashlistStatistics["uploadedTotalLines"] . " Empty lines: " . $hashlistStatistics["uploadedEmptyLines"] . " Valid hashes: " . $hashlistStatistics["uploadedValidHashes"] . " Valid hashes without expected salt: " . $hashlistStatistics["uploadedValidHashesWithoutExpectedSalt"] . " Invalid hashes: " . $hashlistStatistics["uploadedInvalidHashes"]);
 
         NotificationHandler::checkNotifications(DNotificationType::NEW_HASHLIST, new DataSet(array(DPayloadKeys::HASHLIST => $hashlist)));
         break;
     }
+
     Factory::getAgentFactory()->getDB()->commit();
-    return $hashlist;
+    return ["hashlist" => $hashlist, "statistics" => $hashlistStatistics];
   }
 
   /**