diff --git a/backend/api/partners/statistics.py b/backend/api/partners/statistics.py index de6469d660..1db4d54ace 100644 --- a/backend/api/partners/statistics.py +++ b/backend/api/partners/statistics.py @@ -109,11 +109,15 @@ async def get_filtered_statistics( sub_code=MAPSWIPE_GROUP_EMPTY_SUBCODE, message=MAPSWIPE_GROUP_EMPTY_MESSAGE, ) - mapswipe = MapswipeService() - return mapswipe.fetch_filtered_partner_stats( - partner.id, partner.mapswipe_group_id, from_date, to_date - ) + try: + result = await mapswipe.fetch_filtered_partner_stats( + partner.id, partner.mapswipe_group_id, from_date, to_date + ) + finally: + await mapswipe.aclose() + + return result @router.get("/{permalink:str}/general-statistics/") @@ -176,13 +180,16 @@ async def get_general_statistics( ) mapswipe = MapswipeService() - group_dto = mapswipe.fetch_grouped_partner_stats( - partner.id, - partner.mapswipe_group_id, - limit, - offset, - download_as_csv, - ) + try: + group_dto = await mapswipe.fetch_grouped_partner_stats( + partner.id, + partner.mapswipe_group_id, + limit, + offset, + download_as_csv, + ) + finally: + await mapswipe.aclose() if download_as_csv: csv_content = group_dto.to_csv() diff --git a/backend/config.py b/backend/config.py index ec0c811c7a..10415a6243 100644 --- a/backend/config.py +++ b/backend/config.py @@ -269,6 +269,11 @@ def assemble_db_connection( # Sentry backend DSN SENTRY_BACKEND_DSN: Optional[str] = os.getenv("TM_SENTRY_BACKEND_DSN", None) + # Mapswipe backend url + MAPSWIPE_API_URL: str = os.getenv( + "MAPSWIPE_API_URL", "https://backend.mapswipe.org/graphql/" + ) + # Ohsome Stats Token OHSOME_STATS_TOKEN: str = os.getenv("OHSOME_STATS_TOKEN", None) OHSOME_STATS_API_URL: str = os.getenv( diff --git a/backend/services/mapswipe_service.py b/backend/services/mapswipe_service.py index ef7eb7f184..8d412fbe07 100644 --- a/backend/services/mapswipe_service.py +++ b/backend/services/mapswipe_service.py @@ -1,6 +1,7 @@ import json +import logging -import requests +import httpx from backend.exceptions import Conflict from backend.models.dtos.partner_stats_dto import ( @@ -17,10 +18,132 @@ UserGroupMemberDTO, ) -MAPSWIPE_API_URL = "https://api.mapswipe.org/graphql/" +from urllib.parse import urlparse +from backend.config import settings + +logger = logging.getLogger(__name__) + +MAPSWIPE_API_URL = settings.MAPSWIPE_API_URL class MapswipeService: + _client: httpx.AsyncClient | None = None + + async def _ensure_client(self) -> httpx.AsyncClient: + if self._client is None: + self._client = httpx.AsyncClient( + headers={"User-Agent": "mapswipe-backend-service/1.0"}, + timeout=15, + ) + return self._client + + def _get_origin(self) -> str: + parsed = urlparse(MAPSWIPE_API_URL) + return f"{parsed.scheme}://{parsed.netloc}" + + async def _fetch_csrf_cookie(self, client: httpx.AsyncClient) -> str | None: + """GET health-check or root to populate cookies. Return CSRF token if found.""" + origin = self._get_origin() + for path in ("/health-check/", "/"): + try: + await client.get(origin + path) + except httpx.RequestError: + logger.debug("GET %s failed (ignored)", origin + path) + for ck in ("MAPSWIPE-PROD-CSRFTOKEN", "csrftoken", "CSRF-TOKEN"): + val = client.cookies.get(ck) + + if val: + logger.debug("Found CSRF cookie %s", ck) + return val + return None + + async def _post_graphql(self, payload: dict) -> dict: + """ + Async POST GraphQL payload, handle CSRF (Referer/Origin + X-CSRFToken), retry once on 403, + check HTTP and GraphQL errors, and return parsed JSON dict. + """ + client = await self._ensure_client() + origin = self._get_origin() + + # try to obtain CSRF cookie + csrf_token = await self._fetch_csrf_cookie(client) + + headers = { + "Content-Type": "application/json", + "Accept": "application/json", + "Referer": origin, + "Origin": origin, + } + if csrf_token: + headers["X-CSRFToken"] = csrf_token + + try: + resp = await client.post(MAPSWIPE_API_URL, headers=headers, json=payload) + except httpx.RequestError as exc: + logger.exception("MapSwipe POST failed: %s", exc) + raise Conflict( + "MAPSWIPE_NETWORK_ERROR", f"Failed to reach MapSwipe API: {exc}" + ) + + # retry on 403 after refreshing cookies once + if resp.status_code == 403: + logger.warning( + "MapSwipe responded 403; refreshing cookies and retrying once." + ) + await self._fetch_csrf_cookie(client) + # update csrf_token if present + for ck in ("MAPSWIPE-PROD-CSRFTOKEN", "csrftoken", "CSRF-TOKEN"): + csrf_token = client.cookies.get(ck) + if csrf_token: + headers["X-CSRFToken"] = csrf_token + break + else: + headers.pop("X-CSRFToken", None) + + try: + resp = await client.post( + MAPSWIPE_API_URL, headers=headers, json=payload + ) + except httpx.RequestError as exc: + logger.exception("MapSwipe POST retry failed: %s", exc) + raise Conflict( + "MAPSWIPE_NETWORK_ERROR", + f"Failed to reach MapSwipe API on retry: {exc}", + ) + + # HTTP errors + if resp.status_code >= 400: + logger.error( + "MapSwipe HTTP error %s: %s", resp.status_code, resp.text[:2000] + ) + raise Conflict("MAPSWIPE_HTTP_ERROR", f"MapSwipe HTTP {resp.status_code}") + + # parse JSON + try: + parsed = resp.json() + except ValueError: + logger.error("MapSwipe returned non-JSON: %s", resp.text[:2000]) + raise Conflict( + "MAPSWIPE_INVALID_RESPONSE", "MapSwipe returned non-JSON response" + ) + + # GraphQL-level errors + if parsed.get("errors"): + err_msg = parsed["errors"][0].get("message", "GraphQL error") + logger.error( + "MapSwipe GraphQL error: %s -- full: %s", err_msg, parsed["errors"] + ) + raise Conflict( + "MAPSWIPE_GRAPHQL_ERROR", f"MapSwipe GraphQL error: {err_msg}" + ) + + # guard against null data + if parsed.get("data") is None: + logger.error("MapSwipe returned null data: %s", parsed) + raise Conflict("MAPSWIPE_NO_DATA", "MapSwipe returned null data") + + return parsed + @staticmethod def __build_query_user_group_stats(group_id: str, limit: int, offset: int): """A private method to build a graphQl query for fetching a user group's stats from Mapswipe.""" @@ -28,46 +151,49 @@ def __build_query_user_group_stats(group_id: str, limit: int, offset: int): operation_name = "UserGroupStats" query = """ query UserGroupStats($pk: ID!, $limit: Int!, $offset: Int!) { - userGroup(pk: $pk) { - id - userGroupId - name - description - userMemberships(pagination: {limit: $limit, offset: $offset}) { - count - items { - id - userId - username - isActive - totalMappingProjects - totalSwipeTime - totalSwipes - __typename - } - __typename - } - __typename - } - userGroupStats(userGroupId: $pk) { - id - stats { - totalContributors - totalSwipes - totalSwipeTime - __typename - } - statsLatest { - totalContributors - totalSwipeTime - totalSwipes - __typename - } - __typename - } + contributorUserGroup(userGroupId: {firebaseId: $pk}) { + id + name + description + userMemberships(pagination: {limit: $limit, offset: $offset}) { + totalCount + results { + id + isActive + totalSwipes + totalSwipeTime + totalMappingProjects + user { + id + firebaseId + username + __typename + } + __typename + } + __typename + } + __typename + } + communityUserGroupStats(userGroupId: {firebaseId: $pk}) { + id + stats { + totalContributors + totalSwipes + totalSwipeTime + __typename + } + statsLatest { + totalContributors + totalSwipes + totalSwipeTime + __typename + } + __typename + } } """ - variables = {"limit": limit, "offset": offset, "pk": group_id} + variables = {"pk": group_id, "limit": limit, "offset": offset} return {"operationName": operation_name, "query": query, "variables": variables} def __build_query_filtered_user_group_stats( @@ -77,110 +203,94 @@ def __build_query_filtered_user_group_stats( operation_name = "FilteredUserGroupStats" query = """ - query FilteredUserGroupStats($pk: ID!, $fromDate: DateTime!, $toDate: DateTime!) { - userGroup(pk: $pk) { - id - } - userGroupStats(userGroupId: $pk) { - id - filteredStats(dateRange: {fromDate: $fromDate, toDate: $toDate}) { - userStats { - totalMappingProjects - totalSwipeTime - totalSwipes - username - userId - __typename - } - contributionByGeo { - geojson - totalContribution - __typename - } - areaSwipedByProjectType { - totalArea - projectTypeDisplay - projectType - __typename - } - swipeByDate { - taskDate - totalSwipes - __typename - } - swipeTimeByDate { - date - totalSwipeTime - __typename - } - swipeByProjectType { - projectType - projectTypeDisplay - totalSwipes - __typename - } - swipeByOrganizationName { - organizationName - totalSwipes - __typename - } - __typename - } + query FilteredUserGroupStats($pk: ID!, $fromDate: Date!, $toDate: Date!) { + communityUserGroupStats(userGroupId: {firebaseId: $pk}) { + id + filteredStats(dateRange: {fromDate: $fromDate, toDate: $toDate}) { + areaSwipedByProjectType { + projectTypeDisplay + totalArea + projectType + __typename + } + swipeByDate { + taskDate + totalSwipes + __typename + } + swipeByOrganizationName { + totalSwipes + organizationName __typename + } + swipeByProjectGeo { + totalContribution + geojson + __typename + } + swipeByProjectType { + totalSwipes + projectTypeDisplay + projectType + __typename + } + swipeTimeByDate { + totalSwipeTime + date + __typename + } + __typename } + __typename + } } """ - variables = {"fromDate": from_date, "toDate": to_date, "pk": group_id} + variables = {"pk": group_id, "fromDate": from_date, "toDate": to_date} return {"operationName": operation_name, "query": query, "variables": variables} def setup_group_dto( self, partner_id: str, group_id: str, resp_body: str ) -> GroupedPartnerStatsDTO: group_stats = json.loads(resp_body)["data"] - group_dto = GroupedPartnerStatsDTO(provider="mapswipe") - group_dto.id = partner_id - group_dto.id_inside_provider = group_id + group_info = group_stats["contributorUserGroup"] + stats_info = group_stats["communityUserGroupStats"] - if group_stats["userGroup"] is None: + if group_info is None: raise Conflict( "INVALID_MAPSWIPE_GROUP_ID", "The mapswipe group ID linked to this partner is invalid. Please contact an admin.", ) - group_dto.name_inside_provider = group_stats["userGroup"]["name"] - group_dto.description_inside_provider = group_stats["userGroup"]["description"] + group_dto = GroupedPartnerStatsDTO(provider="mapswipe") + group_dto.id = partner_id + group_dto.id_inside_provider = group_id + group_dto.name_inside_provider = group_info["name"] + group_dto.description_inside_provider = group_info["description"] - group_dto.members_count = group_stats["userGroup"]["userMemberships"]["count"] + memberships = group_info["userMemberships"] + group_dto.members_count = memberships["totalCount"] group_dto.members = [] - for user_resp in group_stats["userGroup"]["userMemberships"]["items"]: + for user_resp in memberships["results"]: user = UserGroupMemberDTO() user.id = user_resp["id"] user.is_active = user_resp["isActive"] - user.user_id = user_resp["userId"] - user.username = user_resp["username"] + user.user_id = user_resp["user"]["firebaseId"] + user.username = user_resp["user"]["username"] user.total_contributions = user_resp["totalSwipes"] user.total_contribution_time = user_resp["totalSwipeTime"] user.total_mapping_projects = user_resp["totalMappingProjects"] group_dto.members.append(user) - group_dto.total_contributors = group_stats["userGroupStats"]["stats"][ + group_dto.total_contributors = stats_info["stats"]["totalContributors"] + group_dto.total_contributions = stats_info["stats"]["totalSwipes"] + group_dto.total_contribution_time = stats_info["stats"]["totalSwipeTime"] + group_dto.total_recent_contributors = stats_info["statsLatest"][ "totalContributors" ] - group_dto.total_contributions = group_stats["userGroupStats"]["stats"][ - "totalSwipes" - ] - group_dto.total_contribution_time = group_stats["userGroupStats"]["stats"][ + group_dto.total_recent_contributions = stats_info["statsLatest"]["totalSwipes"] + group_dto.total_recent_contribution_time = stats_info["statsLatest"][ "totalSwipeTime" ] - group_dto.total_recent_contributors = group_stats["userGroupStats"][ - "statsLatest" - ]["totalContributors"] - group_dto.total_recent_contributions = group_stats["userGroupStats"][ - "statsLatest" - ]["totalSwipes"] - group_dto.total_recent_contribution_time = group_stats["userGroupStats"][ - "statsLatest" - ]["totalSwipeTime"] return group_dto @@ -199,16 +309,20 @@ def setup_filtered_dto( filtered_stats_dto.to_date = to_date filtered_stats = json.loads(resp_body)["data"] - if filtered_stats is None or filtered_stats["userGroup"] is None: + if ( + filtered_stats is None + or filtered_stats.get("communityUserGroupStats") is None + ): raise Conflict( "INVALID_MAPSWIPE_GROUP_ID", "The mapswipe group ID linked to this partner is invalid. Please contact an admin.", ) - filtered_stats = filtered_stats["userGroupStats"]["filteredStats"] + filtered_stats = filtered_stats["communityUserGroupStats"]["filteredStats"] - stats_by_user = [] - for user_stats in filtered_stats["userStats"]: + # userStats is not in the updated schema; safely default to empty + filtered_stats_dto.contributions_by_user = [] + for user_stats in filtered_stats.get("userStats", []): user_contributions = UserContributionsDTO() user_contributions.user_id = user_stats["userId"] user_contributions.username = user_stats["username"] @@ -217,11 +331,10 @@ def setup_filtered_dto( user_contributions.total_mapping_projects = user_stats[ "totalMappingProjects" ] - stats_by_user.append(user_contributions) - filtered_stats_dto.contributions_by_user = stats_by_user + filtered_stats_dto.contributions_by_user.append(user_contributions) contributions_by_geo = [] - for geo_stats in filtered_stats["contributionByGeo"]: + for geo_stats in filtered_stats["swipeByProjectGeo"]: geo_contributions = GeoContributionsDTO() geo_contributions.total_contributions = geo_stats["totalContribution"] geojson = GeojsonDTO() @@ -286,7 +399,7 @@ def setup_filtered_dto( filtered_stats_dto.contributions_by_organization_name = organizations return filtered_stats_dto - def fetch_grouped_partner_stats( + async def fetch_grouped_partner_stats( self, partner_id: int, group_id: str, @@ -300,35 +413,28 @@ def fetch_grouped_partner_stats( limit = 1_000_000 offset = 0 - resp_body = requests.post( - MAPSWIPE_API_URL, - headers={"Content-Type": "application/json"}, - data=json.dumps( - self.__build_query_user_group_stats(group_id, limit, offset) - ), - ).text - - group_dto = self.setup_group_dto(partner_id, group_id, resp_body) + payload = self.__build_query_user_group_stats(group_id, limit, offset) + parsed = await self._post_graphql(payload) + group_dto = self.setup_group_dto(partner_id, group_id, json.dumps(parsed)) return group_dto - def fetch_filtered_partner_stats( + async def fetch_filtered_partner_stats( self, partner_id: str, group_id: str, from_date: str, to_date: str, ) -> FilteredPartnerStatsDTO: - resp = requests.post( - MAPSWIPE_API_URL, - headers={"Content-Type": "application/json"}, - data=json.dumps( - self.__build_query_filtered_user_group_stats( - group_id, from_date, to_date - ) - ), + payload = self.__build_query_filtered_user_group_stats( + group_id, from_date, to_date ) - + parsed = await self._post_graphql(payload) filtered_dto = self.setup_filtered_dto( - partner_id, group_id, from_date, to_date, resp.text + partner_id, group_id, from_date, to_date, json.dumps(parsed) ) return filtered_dto + + async def aclose(self): + if self._client: + await self._client.aclose() + self._client = None diff --git a/frontend/src/components/partnerMapswipeStats/swipesByProjectType.js b/frontend/src/components/partnerMapswipeStats/swipesByProjectType.js index 5d79f22cc6..576fc8b92a 100644 --- a/frontend/src/components/partnerMapswipeStats/swipesByProjectType.js +++ b/frontend/src/components/partnerMapswipeStats/swipesByProjectType.js @@ -23,25 +23,25 @@ export const SwipesByProjectType = ({ contributionsByProjectType = [] }) => { backgroundColors = []; contributionsByProjectType.forEach((stat) => { - if (['build_area', 'buildarea'].includes(stat.projectType.toLowerCase())) { + if (['find'].includes(stat.projectType.toLowerCase())) { const contributionsCount = stat.totalcontributions || 0; if (contributionsCount > 0) { chartData.push(contributionsCount); - labelsData.push('Find'); + labelsData.push(stat?.projectTypeDisplay || 'Find'); backgroundColors.push(CHART_COLOURS.orange); } - } else if (['foot_print', 'footprint'].includes(stat.projectType.toLowerCase())) { + } else if (['validate'].includes(stat.projectType.toLowerCase())) { const contributionsCount = stat.totalcontributions || 0; if (contributionsCount > 0) { chartData.push(contributionsCount); - labelsData.push('Validate'); + labelsData.push(stat?.projectTypeDisplay || 'Validate'); backgroundColors.push(CHART_COLOURS.green); } - } else if (['change_detection', 'changedetection'].includes(stat.projectType.toLowerCase())) { + } else if (['street'].includes(stat.projectType.toLowerCase())) { const contributionsCount = stat.totalcontributions || 0; if (contributionsCount > 0) { chartData.push(contributionsCount); - labelsData.push('Compare'); + labelsData.push(stat?.projectTypeDisplay || 'Steet'); backgroundColors.push(CHART_COLOURS.blue); } }