Improve/9 11 23

chrome-extension/public/manifest.json

@@ -2,7 +2,7 @@
   "manifest_version": 3,
   "name": "repoInspector",
   "description": "Inspect repositories",
-  "version": "1.0.1",
+  "version": "1.0.2",
   "action": {
     "default_popup": "popup.html"
   },

chrome-extension/src/features/constants.ts

@@ -1,3 +1,4 @@
 export const USERS_QUERY_LIMIT = 20;
 
 export const MINIMUM_REQUEST_LIMIT_AMOUNT = 10;
+export const MINIMUM_REST_REQUEST_LIMIT_AMOUNT = 20;

chrome-extension/src/features/gql/queries/getForkersQuery.ts

@@ -56,3 +56,27 @@ export const query = gql`
 `;
 
 export const getForkersQuery = print(query);
+
+export const fallbackQuery = gql`
+  query getForkersQuery(
+    $owner: String!
+    $name: String!
+    $cursor: String = null
+    $limit: Int
+  ) {
+    repository(owner: $owner, name: $name) {
+      forks(
+        first: $limit
+        after: $cursor
+        orderBy: { field: CREATED_AT, direction: DESC }
+      ) {
+        pageInfo {
+          endCursor
+          hasNextPage
+        }
+      }
+    }
+  }
+`;
+
+export const getForkersFallbackQuery = print(query);

chrome-extension/src/features/gql/queries/getStargazersQuery.ts

@@ -52,3 +52,27 @@ export const query = gql`
 `;
 
 export const getStargazersQuery = print(query);
+
+export const fallbackQuery = gql`
+  query getStargazersQuery(
+    $owner: String!
+    $name: String!
+    $cursor: String = null
+    $limit: Int
+  ) {
+    repository(owner: $owner, name: $name) {
+      stargazers(
+        first: $limit
+        after: $cursor
+        orderBy: { field: STARRED_AT, direction: DESC }
+      ) {
+        pageInfo {
+          endCursor
+          hasNextPage
+        }
+      }
+    }
+  }
+`;
+
+export const getStargazersFallbackQuery = print(fallbackQuery);

chrome-extension/src/features/repoInspector.ts

@@ -30,8 +30,14 @@ import { NOTIFICATION_TYPES, notificationStore } from './store/notification';
 import { api } from './api';
 import { auth } from './authentication';
 import { historyStore } from './store/history';
-import { MINIMUM_REQUEST_LIMIT_AMOUNT, USERS_QUERY_LIMIT } from './constants';
+import {
+  MINIMUM_REQUEST_LIMIT_AMOUNT,
+  MINIMUM_REST_REQUEST_LIMIT_AMOUNT,
+  USERS_QUERY_LIMIT,
+} from './constants';
 import { calculateTotalRating } from './utils/calculateTotalRating';
+import { getStargazersFallbackQuery } from './gql/queries/getStargazersQuery';
+import { getForkersFallbackQuery } from './gql/queries/getForkersQuery';
 
 let octokit: Octokit;
 
@@ -163,6 +169,8 @@ class RepoInspector {
       const endCursor = resp?.repository?.[type].pageInfo.endCursor;
       const currentRequestItems = resp?.repository?.[type].edges ?? [];
       const requestRemaining = resp?.rateLimit?.remaining;
+      let restRequestRemaining = 5000;
+      let restRateLimitResetAt;
 
       // remove artifacts of graphQL response and normalize data
       const normalizedCurrentRequestItems = await Promise.all(
@@ -174,20 +182,23 @@ class RepoInspector {
 
           if (!mappedItem.login) return {};
 
-          const serializedItem = serializeUser(
+          const { serializedUser, rateLimits } = await serializeUser(
             mappedItem,
             octokit,
             isExtendLocation,
           );
 
-          return serializedItem;
+          restRequestRemaining = Number(rateLimits.rateLimitRemaining);
+          restRateLimitResetAt = rateLimits.rateLimitReset;
+
+          return serializedUser;
         }),
       );
       items = [...items, ...normalizedCurrentRequestItems];
 
       downloaderStore.increaseProgress();
 
-      // if query limits reached - pause inspection
+      // if graphQL query limits reached - pause inspection
       if (
         requestRemaining &&
         requestRemaining <= MINIMUM_REQUEST_LIMIT_AMOUNT
@@ -198,6 +209,21 @@ class RepoInspector {
         return { success: false };
       }
 
+      // if REST query limits reached - pause inspection
+      if (
+        restRequestRemaining &&
+        restRequestRemaining <= MINIMUM_REST_REQUEST_LIMIT_AMOUNT
+      ) {
+        await inspectDataStore.set(inspectDataPropertyName, items as DBUser[]);
+        this._pauseInspection(
+          type,
+          restRateLimitResetAt ?? new Date(),
+          endCursor,
+        );
+
+        return { success: false };
+      }
+
       // if there are more data than we already receive - request for new portion of data
       if (hasNextPage && items.length < max) {
         return await this.getUsers(
@@ -212,6 +238,62 @@ class RepoInspector {
         );
       }
     } catch (error) {
+      const isBadProfileDetected =
+        (error as { errors?: { message: string }[] }).errors?.length &&
+        (error as { errors?: { message: string }[] }).errors?.find(
+          ({ message }) =>
+            message.includes(
+              'Something went wrong while executing your query. Please include',
+            ),
+        );
+
+      if (isBadProfileDetected && limit > 2) {
+        return await this.getUsers(
+          owner,
+          name,
+          type,
+          Math.floor(limit / 2),
+          max,
+          isExtendLocation,
+          items,
+          cursor,
+        );
+      }
+
+      if (isBadProfileDetected && limit <= 2) {
+        const fallbackQuery =
+          type === 'stargazers'
+            ? getStargazersFallbackQuery
+            : getForkersFallbackQuery;
+
+        const resp = await octokit.graphql<
+          GetForkersQueryQuery & GetStargazersQueryQuery
+        >(fallbackQuery, {
+          owner,
+          name,
+          cursor,
+          limit: 1,
+        });
+
+        const hasFallbackNextPage =
+          resp?.repository?.[type].pageInfo.hasNextPage;
+        const fallbackEndCursor = resp?.repository?.[type].pageInfo.endCursor;
+
+        if (hasFallbackNextPage && items.length < max) {
+          return await this.getUsers(
+            owner,
+            name,
+            type,
+            USERS_QUERY_LIMIT,
+            max,
+            isExtendLocation,
+            items,
+            fallbackEndCursor,
+          );
+        }
+      }
+
+      this.alreadyPaused = true;
       await this._stopByError();
     }
 
@@ -266,6 +348,7 @@ class RepoInspector {
         return await this.getStarHistory(owner, name, items, endCursor);
       }
     } catch (error) {
+      this.alreadyPaused = true;
       await this._stopByError();
     }
 
@@ -322,6 +405,7 @@ class RepoInspector {
         return await this.getIssues(owner, name, items, endCursor);
       }
     } catch (error) {
+      this.alreadyPaused = true;
       await this._stopByError();
     }
 
@@ -381,6 +465,7 @@ class RepoInspector {
         return await this.getPullRequests(owner, name, items, endCursor);
       }
     } catch (error) {
+      this.alreadyPaused = true;
       await this._stopByError();
     }
 

chrome-extension/src/features/utils/serializeUser.ts

@@ -26,14 +26,20 @@ export const serializeUser = async (
 
   let event_count;
   let lastEventDate;
+  let rateLimitRemaining;
+  let rateLimitReset;
 
   try {
     // get user event count
     const eventsURL = `https://api.github.com/users/${login}`;
-    const { data } = await octokit.request(
+    const response = await octokit.request(
       `GET ${eventsURL}/events?per_page=100`,
     );
 
+    const { data, headers } = response;
+    rateLimitRemaining = headers['x-ratelimit-remaining'];
+    rateLimitReset = Number(headers['x-ratelimit-reset']) * 1000;
+
     event_count = data.length;
     lastEventDate = data.length && data[0]?.created_at;
   } catch (error) {
@@ -89,5 +95,5 @@ export const serializeUser = async (
     }
   }
 
-  return serializedUser;
+  return { serializedUser, rateLimits: { rateLimitRemaining, rateLimitReset } };
 };

chrome-extension/src/view/popup.vue

@@ -212,8 +212,10 @@ export default {
           Math.ceil(issuesCount / 100) +
           Math.ceil(pullRequestsCount / 100) +
           Math.ceil(stargazerCount / 100) +
-          (settings.stars ? Math.ceil(forks_users / USERS_QUERY_LIMIT) : 0) +
-          (settings.forks ? Math.ceil(stargazer_users / USERS_QUERY_LIMIT) : 0);
+          (settings.stars
+            ? Math.ceil(stargazer_users / USERS_QUERY_LIMIT)
+            : 0) +
+          (settings.forks ? Math.ceil(forks_users / USERS_QUERY_LIMIT) : 0);
 
         this.downloader.stargazers_count = stargazerCount;
         this.downloader.forks_count = forkCount;