Fix #1479

Author: shelld3v

lib/connection/response.py

@@ -31,7 +31,7 @@
     UNKNOWN,
 )
 from lib.parse.url import clean_path, parse_path
-from lib.utils.common import get_readable_size, is_binary, replace_from_all_encodings
+from lib.utils.common import get_readable_size, is_binary, replace_path
 
 
 class BaseResponse:
@@ -42,7 +42,7 @@ def __init__(self, url, response: requests.Response | httpx.Response) -> None:
         self.path = clean_path(self.full_path)
         self.status = response.status_code
         self.headers = response.headers
-        self.redirect = self.headers.get("location") or ""
+        self.redirect = self.headers.get("location", "")
         self.history = [str(res.url) for res in response.history]
         self.content = ""
         self.body = b""
@@ -68,7 +68,7 @@ def size(self) -> str:
     def __hash__(self) -> int:
         # Hash the static parts of the response only.
         # See https://github.com/maurosoria/dirsearch/pull/1436#issuecomment-2476390956
-        body = replace_from_all_encodings(self.content, self.full_path.split("#")[0], "") if self.content else self.body
+        body = replace_path(self.content, self.full_path.split("#")[0], "") if self.content else self.body
         return hash((self.status, body))
 
     def __eq__(self, other: Any) -> bool:

lib/core/scanner.py

@@ -33,7 +33,7 @@
     WILDCARD_TEST_POINT_MARKER,
 )
 from lib.parse.url import clean_path
-from lib.utils.common import replace_from_all_encodings
+from lib.utils.common import replace_path
 from lib.utils.diff import DynamicContentParser, generate_matching_regex
 from lib.utils.random import rand_string
 
@@ -68,7 +68,7 @@ def check(self, path: str, response: BaseResponse) -> bool:
             and we get rid of queries/DOM in path as well because queries in path are usually
             reflected in the redirect as queries too (but we have already got rid of them).
             """
-            redirect = replace_from_all_encodings(
+            redirect = replace_path(
                 clean_path(response.redirect),
                 clean_path(path),
                 REFLECTED_PATH_MARKER,
@@ -111,17 +111,17 @@ def generate_redirect_regex(first_loc: str, first_path: str, second_loc: str, se
 
         How it works:
         1. Replace path in 2 redirect URLs (if it gets reflected in) with a mark
-           (e.g. /path1 -> /foo/path1 and /path2 -> /foo/path2 will become /foo/[mark] for both)
+           (e.g. /path1 -> /foo/path1 and /path2 -> /foo/path2 will become /foo[mark] for both)
         2. Compare 2 redirects and generate a regex that matches both
-           (e.g. /foo/[mark] and /foo/[mark] will have the regex: ^/foo/[mark]$)
+           (e.g. /foo[mark] and /foo[mark] will have the regex: ^/foo[mark]$)
         3. To check if a redirect is wildcard, replace path with the mark and check if it matches this regex
-           (e.g. /path3 -> /bar/path3, the redirect becomes /bar/[mark], which doesn't match the regex ^/foo/[mark]$)
+           (e.g. /path3 -> /bar/path3, the redirect becomes /bar[mark], which doesn't match the regex ^/foo[mark]$)
         """
 
         if first_path:
-            first_loc = first_loc.replace(first_path, REFLECTED_PATH_MARKER)
+            first_loc = first_loc.replace("/" + first_path, REFLECTED_PATH_MARKER)
         if second_path:
-            second_loc = second_loc.replace(second_path, REFLECTED_PATH_MARKER)
+            second_loc = second_loc.replace("/" + second_path, REFLECTED_PATH_MARKER)
 
         return generate_matching_regex(first_loc, second_loc)
 

lib/utils/common.py

@@ -18,6 +18,7 @@
 
 import os
 import sys
+import re
 
 from functools import reduce
 from json import dumps
@@ -136,13 +137,24 @@ def read_stdin():
     return buffer
 
 
-# Replace a substring from an HTML body, where the substring might be encoded
+# Replace a path from an HTML body, where the path might be encoded/decoded
 # in many different ways (URL encoding, HTML escaping, ...).
-def replace_from_all_encodings(string, to_replace, replace_with):
-    string = string.replace(quote(to_replace), replace_with)
-    string = string.replace(quote(quote(to_replace)), replace_with)
-    string = string.replace(unquote(to_replace), replace_with)
-    string = string.replace(unquote(unquote(to_replace)), replace_with)
-    string = string.replace(escape(to_replace), replace_with)
-    string = string.replace(dumps(to_replace), replace_with)
-    return string.replace(to_replace, replace_with)
+#
+# Note:
+# - :path: argument must not start with an "/".
+# - The path in the body followed by an alphanumeric character won't
+# be replaced. For example, "abc" will be replaced from "abc def" but
+# not "abcdef".
+def replace_path(string, path, replace_with):
+    def sub(string, to_replace, replace_with):
+        regex = re.escape(to_replace) + "(?=[^\\w]|$)"
+        return re.sub(to_replace, replace_with, string)
+
+    path = "/" + path
+    string = sub(string, quote(path), replace_with)
+    string = sub(string, quote(quote(path)), replace_with)
+    string = sub(string, unquote(path), replace_with)
+    string = sub(string, unquote(unquote(path)), replace_with)
+    string = sub(string, escape(path), replace_with)
+    string = sub(string, dumps(path), replace_with)
+    return sub(string, path, replace_with)

tests/utils/test_common.py

@@ -18,10 +18,20 @@
 
 from unittest import TestCase
 
-from lib.utils.common import merge_path, strip_and_uniquify, get_valid_filename
+from lib.utils.common import (
+    merge_path,
+    replace_path,
+    strip_and_uniquify,
+    get_valid_filename,
+)
 
 
 class TestCommonUtils(TestCase):
+    def test_replace_path(self):
+        self.assertEqual(replace_path("/abc or /abc?k=v", "abc", "REPLACED"), "REPLACED or REPLACED?k=v", "Path was not replaced")
+        self.assertEqual(replace_path("http://a.co/abc", "abc", "REPLACED"), "http://a.comREPLACED", "Path was not replaced")
+        self.assertEqual(replace_path("http://a.co/abcdef", "abc", "REPLACED"), "http://a.com/abcdef", "Path was replaced eventhough it should have not")
+
     def test_strip_and_uniquify(self):
         self.assertEqual(strip_and_uniquify(["foo", "bar", " bar ", "foo"]), ["foo", "bar"], "The results are not stripped or contain duplicates or in wrong order")