backend: Use planet.osm.org/users_deleted/users_deleted.txt to avoid hammering API

Signed-off-by: Taylor Smock <[email protected]>

Author: tsmock

backend/api/users/resources.py

@@ -1,5 +1,6 @@
 from distutils.util import strtobool
 from typing import Optional
+from collections.abc import Generator
 
 from flask import stream_with_context, Response
 from flask_restful import Resource, current_app, request
@@ -146,22 +147,36 @@ def get(self):
     @token_auth.login_required
     def delete(self):
         if UserService.is_user_an_admin(token_auth.current_user()):
-
-            def delete_users():
-                for user in User.get_all_users_not_paginated():
-                    # We specifically want to remove users that have deleted their OSM accounts.
-                    if OSMService.is_osm_user_gone(user.id):
-                        data = UserService.delete_user_by_id(
-                            user.id, token_auth.current_user()
-                        ).to_primitive()
-                        yield f"\u001e{data}\n"
-
             return Response(
-                stream_with_context(delete_users()),
+                stream_with_context(UsersAllAPI._delete_users()),
                 headers={"Content-Type": "application/json-seq"},
             )
         raise Unauthorized()
 
+    @staticmethod
+    def _delete_users() -> Generator[str, None, None]:
+        # Updated daily
+        deleted_users = OSMService.get_deleted_users()
+        if deleted_users:
+            last_deleted_user = 0
+            for user in User.get_all_users_not_paginated(User.id):
+                while last_deleted_user < user.id:
+                    last_deleted_user = next(deleted_users)
+                if last_deleted_user == user.id:
+                    data = UserService.delete_user_by_id(
+                        user.id, token_auth.current_user()
+                    ).to_primitive()
+                    yield f"\u001e{data}\n"
+            return
+        # Fall back to hitting the API (if the OSM API is not the default)
+        for user in User.get_all_users_not_paginated():
+            # We specifically want to remove users that have deleted their OSM accounts.
+            if OSMService.is_osm_user_gone(user.id):
+                data = UserService.delete_user_by_id(
+                    user.id, token_auth.current_user()
+                ).to_primitive()
+                yield f"\u001e{data}\n"
+
 
 class UsersQueriesUsernameAPI(Resource):
     @token_auth.login_required

backend/models/postgis/user.py

@@ -182,9 +182,12 @@ def get_all_users(query: UserSearchQuery) -> UserSearchDTO:
         return dto
 
     @staticmethod
-    def get_all_users_not_paginated():
+    def get_all_users_not_paginated(*order):
         """Get all users in DB"""
-        return db.session.query(User.id).all()
+        query = db.session.query(User.id)
+        if not order:
+            return query.all()
+        return query.order_by(*order).all()
 
     @staticmethod
     def filter_users(user_filter: str, project_id: int, page: int) -> UserFilterDTO:

backend/services/users/osm_service.py

@@ -1,3 +1,7 @@
+import re
+from collections.abc import Generator
+from typing import Optional
+
 import requests
 from flask import current_app
 
@@ -32,6 +36,32 @@ def is_osm_user_gone(user_id: int) -> bool:
 
         return False
 
+    @staticmethod
+    def get_deleted_users() -> Optional[Generator[int, None, None]]:
+        """
+        Get the list of deleted users from OpenStreetMap.
+        This only returns users from the https://www.openstreetmap.org instance.
+        :return: The deleted users
+        """
+        if current_app.config["OSM_SERVER_URL"] == "https://www.openstreetmap.org":
+
+            def get_planet_osm_deleted_users() -> Generator[int, None, None]:
+                response = requests.get(
+                    "https://planet.openstreetmap.org/users_deleted/users_deleted.txt",
+                    stream=True,
+                )
+                username = re.compile(r"^\s*(\d+)\s*$")
+                try:
+                    for line in response.iter_lines(decode_unicode=True):
+                        match = username.fullmatch(line)
+                        if match:
+                            yield int(match.group(1))
+                finally:
+                    response.close()
+
+            return get_planet_osm_deleted_users()
+        return None
+
     @staticmethod
     def get_osm_details_for_user(user_id: int) -> UserOSMDTO:
         """

tests/backend/integration/services/users/test_osm_service.py

@@ -17,3 +17,15 @@ def test_get_osm_details_for_user_returns_user_details_if_valid_user_id(self):
     def test_is_user_deleted(self):
         self.assertTrue(OSMService.is_osm_user_gone(535043))
         self.assertFalse(OSMService.is_osm_user_gone(2078753))
+
+    def test_get_deleted_users(self):
+        # These are the first 10 deleted users on 2024-04-16. This should ensure that the test finishes quickly.
+        # Otherwise, it can take 6s+ (dependent upon network speed)
+        deleted_users = [4, 142, 593, 601, 1769, 2161, 2238, 2782, 2868]
+        generator = OSMService.get_deleted_users()
+        for deleted_user in generator:
+            if deleted_user in deleted_users:
+                deleted_users.remove(deleted_user)
+            if len(deleted_users) == 0:
+                break
+        self.assertEquals(0, len(deleted_users))