diff --git a/ci/apiv2/hashtopolis.py b/ci/apiv2/hashtopolis.py index 1d98f8735..dde1de2b9 100644 --- a/ci/apiv2/hashtopolis.py +++ b/ci/apiv2/hashtopolis.py @@ -1023,11 +1023,13 @@ def export_wordlist(self, hashlist): response = self._helper_request("exportWordlist", payload) return File(**response['data']) - def import_cracked_hashes(self, hashlist, source_data: str, separator): + def import_cracked_hashes(self, hashlist, source_type, source_data: str, separator, overwrite): payload = { 'hashlistId': hashlist.id, + 'sourceType': source_type, 'sourceData': base64.b64encode(source_data.encode()).decode(), 'separator': separator, + 'overwrite': overwrite, } response = self._helper_request("importCrackedHashes", payload) return response['meta'] diff --git a/ci/apiv2/test_hashlist.py b/ci/apiv2/test_hashlist.py index 401be59ac..186c0a25b 100644 --- a/ci/apiv2/test_hashlist.py +++ b/ci/apiv2/test_hashlist.py @@ -68,7 +68,7 @@ def test_export_wordlist(self): cracked = "cc03e747a6afbbcbf8be7668acfebee5:test123" helper = Helper() - helper.import_cracked_hashes(model_obj, cracked, ':') + helper.import_cracked_hashes(model_obj, 'paste', cracked, ':', 0) file = helper.export_wordlist(model_obj) @@ -84,7 +84,7 @@ def test_import_cracked_hashes(self): cracked = "cc03e747a6afbbcbf8be7668acfebee5:test123" helper = Helper() - result = helper.import_cracked_hashes(model_obj, cracked, ':') + result = helper.import_cracked_hashes(model_obj, 'paste', cracked, ':', 0) self.assertEqual(result['totalLines'], 1) self.assertEqual(result['newCracked'], 1) @@ -98,7 +98,7 @@ def test_import_cracked_hashes_invalid(self): cracked = "cc03e747a6afbbcbf8be7668acfebee5__test123" helper = Helper() - result = helper.import_cracked_hashes(model_obj, cracked, ':') + result = helper.import_cracked_hashes(model_obj, 'paste', cracked, ':', 0) self.assertEqual(result['totalLines'], 1) self.assertEqual(result['invalid'], 1) @@ -112,7 +112,7 @@ def test_import_cracked_hashes_notfound(self): cracked = "ffffffffffffffffffffffffffffffff:test123" helper = Helper() - result = helper.import_cracked_hashes(model_obj, cracked, ':') + result = helper.import_cracked_hashes(model_obj, 'paste', cracked, ':', 0) self.assertEqual(result['totalLines'], 1) self.assertEqual(result['notFound'], 1) @@ -126,9 +126,9 @@ def test_import_cracked_hashes_already_cracked(self): cracked = "cc03e747a6afbbcbf8be7668acfebee5:test123" helper = Helper() - helper.import_cracked_hashes(model_obj, cracked, ':') + helper.import_cracked_hashes(model_obj, 'paste', cracked, ':', 0) - result = helper.import_cracked_hashes(model_obj, cracked, ':') + result = helper.import_cracked_hashes(model_obj, 'paste', cracked, ':', 0) self.assertEqual(result['totalLines'], 1) self.assertEqual(result['alreadyCracked'], 1) diff --git a/ci/tests/HashlistTest.class.php b/ci/tests/HashlistTest.class.php index cb7d2e4c6..b799b402c 100644 --- a/ci/tests/HashlistTest.class.php +++ b/ci/tests/HashlistTest.class.php @@ -109,6 +109,7 @@ private function testImportCracked() { "request" => "importCracked", "hashlistId" => 1, "separator" => ":", + "overwrite" => 0, // sending 3 founds of the hashlist "data" => "MDAyODA4MGU3ZmE4YzgxMjY4ZWYzNDBkN2Q2OTI2ODE6Zm91bmQxCjAwMmU5NWQ4MmJlMzAzOTZmY2NkMzc1ZmYyM2Y4YjRjOmZvdW5kMgowMDM0YzVlNDE4YWU0ZjJlYmE1OTBhMTY2OTZlZGJiMzpmb3VuZDM=", "accessKey" => "mykey" diff --git a/src/inc/apiv2/helper/importCrackedHashes.routes.php b/src/inc/apiv2/helper/importCrackedHashes.routes.php index 5c2ca9ee7..b5fc01d4b 100644 --- a/src/inc/apiv2/helper/importCrackedHashes.routes.php +++ b/src/inc/apiv2/helper/importCrackedHashes.routes.php @@ -26,8 +26,10 @@ public function getRequiredPermissions(string $method): array { public function getFormFields(): array { return [ Hashlist::HASHLIST_ID => ["type" => "int"], + "sourceType" => ['type' => 'str'], "sourceData" => ['type' => 'str'], "separator" => ['type' => 'str'], + "overwrite" => ['type' => 'int'], ]; } @@ -46,13 +48,38 @@ public static function getResponse(): array { /** * Endpoint to import cracked hashes into a hashlist. * @throws HTException + * @throws HttpError */ public function actionPost($data): object|array|null { $hashlist = self::getHashlist($data[Hashlist::HASHLIST_ID]); - $importData = base64_decode($data["sourceData"]); + // Cast to processZap compatible upload format + $dummyPost = []; + switch ($data["sourceType"]) { + case "paste": + $dummyPost["hashfield"] = base64_decode($data["sourceData"]); + break; + case "import": + $dummyPost["importfile"] = $data["sourceData"]; + break; + case "url": + $dummyPost["url"] = $data["sourceData"]; + break; + default: + // TODO: Choice validation are model based checks + throw new HttpErrorException("sourceType value '" . $data["sourceType"] . "' is not supported (choices paste, import, url"); + } + + if ($data["sourceType"] == "paste") { + if (strlen($data["sourceData"]) == 0) { + throw new HttpError("sourceType=paste, requires sourceData to be non-empty"); + } + else if ($dummyPost["hashfield"] === false) { + throw new HttpError("sourceData not valid base64 encoding"); + } + } - $result = HashlistUtils::processZap($hashlist->getId(), $data["separator"], "paste", ["hashfield" => $importData], [], $this->getCurrentUser()); + $result = HashlistUtils::processZap($hashlist->getId(), $data["separator"], $data["sourceType"], $dummyPost, [], $this->getCurrentUser(), (isset($data["overwrite"]) && intval($data["overwrite"]) == 1) ? true : false); return [ "totalLines" => $result[0], diff --git a/src/inc/apiv2/model/hashlists.routes.php b/src/inc/apiv2/model/hashlists.routes.php index 99eda1fe6..d50f9382f 100644 --- a/src/inc/apiv2/model/hashlists.routes.php +++ b/src/inc/apiv2/model/hashlists.routes.php @@ -102,6 +102,7 @@ public function getFormFields(): array { /** * @throws HttpErrorException + * @throws HttpError * @throws HTException */ protected function createObject(array $data): int { @@ -122,11 +123,12 @@ protected function createObject(array $data): int { throw new HttpErrorException("sourceType value '" . $data["sourceType"] . "' is not supported (choices paste, import, url"); } - // TODO: validate input is valid base64 encoded if ($data["sourceType"] == "paste") { if (strlen($data["sourceData"]) == 0) { - // TODO: Should be 400 instead - throw new HttpErrorException("sourceType=paste, requires sourceData to be non-empty"); + throw new HttpError("sourceType=paste, requires sourceData to be non-empty"); + } + else if ($dummyPost["hashfield"] === false) { + throw new HttpError("sourceData not valid base64 encoding"); } } diff --git a/src/inc/handlers/HashlistHandler.class.php b/src/inc/handlers/HashlistHandler.class.php index 1e2672282..05024e0c5 100644 --- a/src/inc/handlers/HashlistHandler.class.php +++ b/src/inc/handlers/HashlistHandler.class.php @@ -49,7 +49,7 @@ public function handle($action) { break; case DHashlistAction::PROCESS_ZAP: AccessControl::getInstance()->checkPermission(DHashlistAction::PROCESS_ZAP_PERM); - $data = HashlistUtils::processZap($_POST['hashlist'], $_POST['separator'], $_POST['source'], $_POST, $_FILES, AccessControl::getInstance()->getUser()); + $data = HashlistUtils::processZap($_POST['hashlist'], $_POST['separator'], $_POST['source'], $_POST, $_FILES, AccessControl::getInstance()->getUser(), (isset($_POST["overwrite"]) && intval($_POST["overwrite"]) == 1) ? true : false); UI::addMessage(UI::SUCCESS, "Processed pre-cracked hashes: " . $data[0] . " total lines, " . $data[1] . " new cracked hashes, " . $data[2] . " were already cracked, " . $data[3] . " invalid lines, " . $data[4] . " not matching entries (" . $data[5] . "s)!"); if ($data[6] > 0) { UI::addMessage(UI::WARN, $data[6] . " entries with too long plaintext"); diff --git a/src/inc/user-api/UserAPIHashlist.class.php b/src/inc/user-api/UserAPIHashlist.class.php index 472298bae..affacb26f 100644 --- a/src/inc/user-api/UserAPIHashlist.class.php +++ b/src/inc/user-api/UserAPIHashlist.class.php @@ -182,7 +182,8 @@ private function importCracked($QUERY) { 'paste', ['hashfield' => base64_decode($QUERY[UQueryHashlist::HASHLIST_DATA])], [], - $this->user + $this->user, + false ); $response = [ UResponseHashlist::SECTION => $QUERY[UQueryHashlist::SECTION], diff --git a/src/inc/utils/HashlistUtils.class.php b/src/inc/utils/HashlistUtils.class.php index a3d4d8127..90d22270c 100644 --- a/src/inc/utils/HashlistUtils.class.php +++ b/src/inc/utils/HashlistUtils.class.php @@ -305,10 +305,11 @@ public static function rename($hashlistId, $name, $user) { * @param array $post * @param array $files * @param User $user + * @param boolean $overwritePlaintext * @return int[] * @throws HTException */ - public static function processZap($hashlistId, $separator, $source, $post, $files, $user) { + public static function processZap($hashlistId, $separator, $source, $post, $files, $user, $overwritePlaintext) { // pre-crack hashes processor $hashlist = HashlistUtils::getHashlist($hashlistId); if (!AccessUtils::userCanAccessHashlists($hashlist, $user)) { @@ -429,16 +430,23 @@ public static function processZap($hashlistId, $separator, $source, $post, $file } else if ($hashEntry->getIsCracked() == 1) { $alreadyCracked++; - continue; + if (!$overwritePlaintext) { + continue; + } } $plain = str_replace($hash . $separator . $hashEntry->getSalt() . $separator, "", $data); if (strlen($plain) > SConfig::getInstance()->getVal(DConfig::PLAINTEXT_MAX_LENGTH)) { $tooLong++; continue; } + + if ($hashEntry->getIsCracked() != 1) { + $newCracked++; + $crackedIn[$hashEntry->getHashlistId()]++; + } + $hashFactory->mset($hashEntry, [Hash::PLAINTEXT => $plain, Hash::IS_CRACKED => 1, Hash::TIME_CRACKED => time()]); - $newCracked++; - $crackedIn[$hashEntry->getHashlistId()]++; + if ($hashlist->getFormat() == DHashlistFormat::PLAIN) { $zaps[] = new Zap(null, $hashEntry->getHash(), time(), null, $hashlist->getId()); } @@ -471,19 +479,28 @@ public static function processZap($hashlistId, $separator, $source, $post, $file foreach ($hashEntries as $hashEntry) { if ($hashEntry->getIsCracked() == 1) { $alreadyCracked++; - continue; + if (!$overwritePlaintext) { + continue; + } } + $plain = str_replace($hash . $separator, "", $data); + if (strlen($plain) > SConfig::getInstance()->getVal(DConfig::PLAINTEXT_MAX_LENGTH)) { $tooLong++; continue; } + + if ($hashEntry->getIsCracked() != 1) { + $newCracked++; + $crackedIn[$hashEntry->getHashlistId()]++; + } + $hashFactory->mset($hashEntry, [Hash::PLAINTEXT => $plain, Hash::IS_CRACKED => 1, Hash::TIME_CRACKED => time()]); - $crackedIn[$hashEntry->getHashlistId()]++; + if ($hashlist->getFormat() == DHashlistFormat::PLAIN) { $zaps[] = new Zap(null, $hashEntry->getHash(), time(), null, $hashlist->getId()); } - $newCracked++; } } $bufferCount++;