Merge branch 'nanos:main' into main

Author: thoresson

.github/workflows/get_context.yml

@@ -22,7 +22,7 @@ jobs:
          cache: 'pip' # caching pip dependencies
       - run: pip install -r requirements.txt
       - name: Download all workflow run artifacts
-        uses: dawidd6/action-download-artifact@v3
+        uses: dawidd6/action-download-artifact@v6
         with:
           name: artifacts
           workflow: get_context.yml
@@ -37,7 +37,8 @@ jobs:
           name: artifacts
           path: |
             artifacts
-      - name: Checkout user's forked repository for keeping workflow alive
-        uses: actions/checkout@v4
-      - name: Keep workflow alive
-        uses: gautamkrishnar/keepalive-workflow@v1
+      # Temporarily disable these while we figure out what to do.
+      # - name: Checkout user's forked repository for keeping workflow alive
+      #   uses: actions/checkout@v4
+      # - name: Keep workflow alive
+      #   uses: gautamkrishnar/keepalive-workflow@v1

.github/workflows/update-version-number.yaml

@@ -18,12 +18,14 @@ jobs:
           git config user.email bot@thms.uk
           VERSION=${{ github.ref_name }}
           VERSION="${VERSION:1}"
-          sed -i -E 's/VERSION = "[0-9]+.[0-9]+.[0-9]+"/VERSION = "'$VERSION'"/' find_posts.py
-          git add find_posts.py 
-          git commit -m "[bot] Bump version to $VERSION"
-          git push origin HEAD:main > /dev/null 2>&1
-          git tag -f "v$VERSION"
-          git push -f origin "v$VERSION" > /dev/null 2>&1
+          if ! grep "VERSION = \"$VERSION\"" "find_posts.py"; then
+            sed -i -E 's/VERSION = "[0-9]+.[0-9]+.[0-9]+"/VERSION = "'$VERSION'"/' find_posts.py
+            git add find_posts.py 
+            git commit -m "[bot] Bump version to $VERSION"
+            git push origin HEAD:main > /dev/null 2>&1
+            git tag -f "v$VERSION"
+            git push -f origin "v$VERSION" > /dev/null 2>&1
+          fi
   
   docker:
     runs-on: ubuntu-latest

find_posts.py

@@ -20,7 +20,7 @@
 logger = logging.getLogger("FediFetcher")
 robotParser = urllib.robotparser.RobotFileParser()
 
-VERSION = "7.1.12"
+VERSION = "7.1.16"
 
 argparser=argparse.ArgumentParser()
 
@@ -367,19 +367,12 @@ def get_timeline(server, access_token, max):
 
         if response.status_code == 200:
             toots = response.json()
-        elif response.status_code == 401:
-            raise Exception(
-                f"Error getting URL {url}. Status code: {response.status_code}. "
-                "Ensure your access token is correct"
-            )
-        elif response.status_code == 403:
-            raise Exception(
-                f"Error getting URL {url}. Status code: {response.status_code}. "
-                "Make sure you have the read:statuses scope enabled for your access token."
-            )
         else:
-            raise Exception(
-                f"Error getting URL {url}. Status code: {response.status_code}"
+            report_mastodon_error(
+                f"Error getting URL {url}",
+                response.status_code,
+                access_token,
+                "read:statuses"
             )
 
         # Paginate as needed
@@ -401,19 +394,12 @@ def get_toots(url, access_token):
 
     if response.status_code == 200:
         return response
-    elif response.status_code == 401:
-        raise Exception(
-            f"Error getting URL {url}. Status code: {response.status_code}. "
-            "It looks like your access token is incorrect."
-        )
-    elif response.status_code == 403:
-        raise Exception(
-            f"Error getting URL {url}. Status code: {response.status_code}. "
-            "Make sure you have the read:statuses scope enabled for your access token."
-        )
     else:
-        raise Exception(
-            f"Error getting URL {url}. Status code: {response.status_code}"
+        report_mastodon_error(
+            f"Error getting URL {url}",
+            response.status_code,
+            access_token,
+            "read:statuses"
         )
 
 def get_active_user_ids(server, access_token, reply_interval_hours):
@@ -432,19 +418,12 @@ def get_active_user_ids(server, access_token, reply_interval_hours):
                 if last_active > since:
                     logger.info(f"Found active user: {user['username']}")
                     yield user["id"]
-    elif resp.status_code == 401:
-        raise Exception(
-            f"Error getting user IDs on server {server}. Status code: {resp.status_code}. "
-            "Ensure your access token is correct"
-        )
-    elif resp.status_code == 403:
-        raise Exception(
-            f"Error getting user IDs on server {server}. Status code: {resp.status_code}. "
-            "Make sure you have the admin:read:accounts scope enabled for your access token."
-        )
     else:
-        raise Exception(
-            f"Error getting user IDs on server {server}. Status code: {resp.status_code}"
+        report_mastodon_error(
+            f"Error getting user IDs on server {server}",
+            resp.status_code,
+            access_token,
+            "admin:read:accounts"
         )
 
 
@@ -491,16 +470,14 @@ def get_reply_toots(user_id, server, access_token, seen_urls, reply_since):
         for toot in toots:
             logger.debug(f"Found reply toot: {toot['url']}")
         return toots
-    elif resp.status_code == 403:
-        raise Exception(
-            f"Error getting replies for user {user_id} on server {server}. Status code: {resp.status_code}. "
-            "Make sure you have the read:statuses scope enabled for your access token."
+    else:
+        report_mastodon_error(
+            f"Error getting replies for user {user_id} on server {server}",
+            resp.status_code,
+            access_token,
+            "read:statuses"
         )
 
-    raise Exception(
-        f"Error getting replies for user {user_id} on server {server}. Status code: {resp.status_code}"
-    )
-
 
 def toot_context_can_be_fetched(toot):
     fetchable = toot["visibility"] in ["public", "unlisted"]
@@ -651,6 +628,11 @@ def parse_url(url, parsed_urls):
         if match is not None:
             parsed_urls[url] = match
 
+    if url not in parsed_urls:
+        match = parse_pleroma_uri(url)
+        if match is not None:
+            parsed_urls[url] = match
+
     if url not in parsed_urls:
         match = parse_lemmy_url(url)
         if match is not None:
@@ -719,6 +701,13 @@ def parse_pleroma_url(url):
         return None
     return None
 
+def parse_pleroma_uri(uri):
+    """parse a Pleroma URL and return the server and ID"""
+    match = re.match(r"https://(?P<server>[^/]+)/notice/(?P<toot_id>[^/]+)", uri)
+    if match is not None:
+        return (match.group("server"), match.group("toot_id"))
+    return None
+
 def parse_pleroma_profile_url(url):
     """parse a Pleroma Profile URL and return the server and username"""
     match = re.match(r"https://(?P<server>[^/]+)/users/(?P<username>[^/]+)", url)
@@ -1033,20 +1022,11 @@ def get_paginated_mastodon(url, max, headers = {}, timeout = 0, max_tries = 5):
     response = get(furl, headers, timeout, max_tries)
 
     if response.status_code != 200:
-        if response.status_code == 401:
-            raise Exception(
-                f"Error getting URL {url}. Status code: {response.status_code}. "
-                "Ensure your access token is correct"
-            )
-        elif response.status_code == 403:
-            raise Exception(
-                f"Error getting URL {url}. Status code: {response.status_code}. "
-                "Make sure you have the correct scopes enabled for your access token."
-            )
-        else:
-            raise Exception(
-                f"Error getting URL {url}. Status code: {response.status_code}"
-            )
+        report_mastodon_error(
+            f"Error getting URL {url}",
+            response.status_code,
+            headers.get('Authorization', '').replace("Bearer ", ""),
+        )
 
     result = response.json()
 
@@ -1470,6 +1450,22 @@ def fetch_timeline_context(timeline_posts, token, parsed_urls, seen_hosts, seen_
 
         add_user_posts(arguments.server, token, filter_known_users(mentioned_users, all_known_users), recently_checked_users, all_known_users, seen_urls, seen_hosts)
 
+def report_mastodon_error(error_message, error_code, access_token, required_scope = ''):
+    subline = ""
+    match error_code:
+        case 401:
+            subline = "\nIt looks like your access token is incorrect. Consider generating a new access token, and/or ensure you have copy and pasted the whole token correctly."
+        case 403:
+            if(required_scope != ""):
+                subline = f"\nAdd the {required_scope} scope to your access token, and regenerate the token."
+            else:
+                subline = "\nMake sure you have enabled the required scope(s) for your token."
+
+    raise Exception(
+        f"{error_message} with token {access_token[:+5]}{'*' * (len(access_token) - 10)}{access_token[-5:]}. Status code: {error_code} "
+        f"{subline}"
+    )
+
 if __name__ == "__main__":
     start = datetime.now()
 
@@ -1519,7 +1515,7 @@ def fetch_timeline_context(timeline_posts, token, parsed_urls, seen_hosts, seen_
     if tokens := [token for envvar, token in os.environ.items() if envvar.lower().startswith("ff_access_token")]:
         arguments.access_token = tokens
 
-    logger.info("Starting FediFetcher")
+    logger.info(f"Starting FediFetcher v{VERSION}")
 
     if(arguments.server == None or arguments.access_token == None):
         logger.critical("You must supply at least a server name and an access token")

requirements.txt

@@ -8,7 +8,7 @@ packaging==24.1
 pluggy==1.5.0
 pytest==8.2.2
 python-dateutil==2.8.2
-requests==2.32.0
+requests==2.32.3
 six==1.16.0
 smmap==5.0.0
 urllib3==1.26.19

tests/test_find_posts.py

@@ -39,9 +39,11 @@
     parse_pixelfed_profile_url,
     parse_pixelfed_url,
     parse_pleroma_url,
+    parse_pleroma_uri,
     post,
     set_server_apis,
     user_has_opted_out,
+    parse_url
 )
 
 
@@ -874,6 +876,10 @@ def test_parse_pleroma_url(mock_get_redirect_url):
     result = parse_pleroma_url("https://different.example.com/objects/111")
     assert result == ("different.example.com", "789")
 
+def test_parse_pleroma_uri():
+    # Test that a valid URI is correctly parsed
+    uri = "https://friedcheese.us/notice/Arv4zBVnAR84mmkVay"
+    assert parse_pleroma_uri(uri) == ("friedcheese.us", "Arv4zBVnAR84mmkVay")    
 
 import re
 import pytest
@@ -951,6 +957,25 @@ def test_parse_peertube_url_valid():
     # assert that the result is as expected
     assert result == expected
 
+def test_parse_url():
+    tests = [
+        (
+            "https://video.infosec.exchange/videos/watch/56f1d0b5-d98f-4bad-b1e7-648ae074ab9d",
+            ("video.infosec.exchange", "56f1d0b5-d98f-4bad-b1e7-648ae074ab9d")
+        ),
+        (
+            "https://veedeo.org/videos/watch/a51bb77c-e1bd-4d6a-b119-95af176f6d66",
+            ("veedeo.org", "a51bb77c-e1bd-4d6a-b119-95af176f6d66")
+        ),
+        (
+            'https://foo.bar/nothing',
+            None
+        )
+    ]
+    for (url,expected) in tests:
+        result = parse_url(url, {})
+        assert result == expected
+
 
 def test_parse_peertube_url_invalid():
     # define an invalid url