links: fix url joining

Author: chapmanjacobd

library/utils/web.py

@@ -639,9 +639,11 @@ def construct_absolute_url(base_url, href):
         return href
 
     if not up.netloc:
-        if not base_url.endswith("/") and not href.startswith("/"):
-            base_url += "/"
-
+        base_parsed = urlparse(base_url)
+        path = base_parsed.path
+        if not path.endswith("/") and path != "":
+            path = path.rsplit("/", 1)[0] + "/"
+        base_url = base_parsed._replace(path=path).geturl()
         href = urljoin(base_url, href)
 
     if href.startswith("//"):

tests/utils/test_web.py

@@ -4,7 +4,7 @@
 from bs4 import BeautifulSoup
 
 from library.utils.path_utils import safe_unquote
-from library.utils.web import WebPath, extract_nearby_text, url_encode, url_to_local_path
+from library.utils.web import WebPath, construct_absolute_url, extract_nearby_text, url_encode, url_to_local_path
 from tests.utils import p
 
 
@@ -250,3 +250,22 @@ def test_parent_property():
     assert str(web_path.parent.parent.parent.parent.parent.parent) == "https://<netloc>/<path1>"
     assert str(web_path.parent.parent.parent.parent.parent.parent.parent) == "https://<netloc>"
     assert str(web_path.parent.parent.parent.parent.parent.parent.parent.parent) == "https://<netloc>"
+
+@pytest.mark.parametrize(
+    "base_url, href, expected",
+    [
+        ("https://unli.xyz/diskprices/index.html", "./ch/", "https://unli.xyz/diskprices/ch/"),
+        ("https://unli.xyz/diskprices/index.html", "ch/", "https://unli.xyz/diskprices/ch/"),
+        ("https://unli.xyz/diskprices/index.html", "/ch/", "https://unli.xyz/ch/"),
+        ("https://unli.xyz/diskprices/", "ch/", "https://unli.xyz/diskprices/ch/"),
+        ("https://unli.xyz/diskprices", "ch/", "https://unli.xyz/ch/"),
+        ("https://unli.xyz", "diskprices/ch/", "https://unli.xyz/diskprices/ch/"),
+        ("https://unli.xyz", "/ch/", "https://unli.xyz/ch/"),
+        ("https://unli.xyz/", "//example.com/ch/", "https://example.com/ch/"),
+        ("https://unli.xyz/diskprices", "ftp://example.com/ch/", "ftp://example.com/ch/"),
+        ("https://unli.xyz/diskprices", "ssh://example.com/ch/", "ssh://example.com/ch/"),
+    ]
+)
+def test_construct_absolute_url(base_url, href, expected):
+    result = construct_absolute_url(base_url, href)
+    assert result == expected