Added GitHubController, updated core version,

removed RestOperationsSupplier to fix restClient implementation,
logging improvements with total record count,

Author: danielyhuang

pom.xml

@@ -50,7 +50,7 @@
     <repository.name>hygieia-scm-github-graphql-collector</repository.name>
     <apache.rat.plugin.version>0.13</apache.rat.plugin.version>
     <bc.version>3.0.1</bc.version>
-    <com.capitalone.dashboard.core.version>3.7.24</com.capitalone.dashboard.core.version>
+    <com.capitalone.dashboard.core.version>3.9.5</com.capitalone.dashboard.core.version>
     <commons.codec.version>1.14</commons.codec.version>
     <commons.io.version>2.4</commons.io.version>
     <commons.lang.version>3.10</commons.lang.version>

src/main/java/com/capitalone/dashboard/collector/GitHubCollectorTask.java

@@ -168,23 +168,22 @@ public void collect(Collector collector){
                enabledRepos = enabledRepos.stream().filter(repo -> GithubRepoMatcher.orgNameMatcher(repo.getRepoUrl(), searchCriteria[1])).collect(Collectors.toList());
            }
        }
+       LOG.info("GitHubCollectorTask:collect start, total enabledRepos=" + enabledRepos.size());
+       LOG.warn("error threshold = " + gitHubSettings.getErrorThreshold());
        collectProcess(collector, enabledRepos );
+
    }
 
 
 
     @SuppressWarnings({"PMD.AvoidDeeplyNestedIfStmts"})
     public void collectProcess(Collector collector, List<GitHubRepo> enabledRepos) {
-
-        logBanner("Starting...");
         long start = System.currentTimeMillis();
         int repoCount = 0;
         int commitCount = 0;
         int pullCount = 0;
         int issueCount = 0;
 
-        LOG.info("GitHubCollectorTask:collect start, total enabledRepos=" + enabledRepos.size());
-        LOG.warn("error threshold = " + gitHubSettings.getErrorThreshold());
         for (GitHubRepo repo : enabledRepos) {
             repoCount++;
             String repoUrl = repo==null?"null":(repo.getRepoUrl() + "/tree/" + repo.getBranch());
@@ -281,6 +280,8 @@ public void collectProcess(Collector collector, List<GitHubRepo> enabledRepos) {
         long elapsedSeconds = (end - start) / 1000;
         LOG.info(String.format("GitHubCollectorTask:collect stop, totalProcessSeconds=%d, totalRepoCount=%d, totalNewPulls=%d, totalNewCommits=%d totalNewIssues=%d",
                 elapsedSeconds, repoCount, pullCount, commitCount, issueCount));
+
+        collector.setLastExecutionRecordCount(repoCount+pullCount+commitCount+issueCount);
     }
 
 

src/main/java/com/capitalone/dashboard/collector/RestOperationsSupplier.java

@@ -0,0 +1,64 @@
+package com.capitalone.dashboard.controller;
+
+import com.capitalone.dashboard.collector.GitHubCollectorTask;
+import com.capitalone.dashboard.misc.HygieiaException;
+import com.capitalone.dashboard.model.Collector;
+import com.capitalone.dashboard.model.GitHubRepo;
+import com.capitalone.dashboard.repository.BaseCollectorRepository;
+import com.capitalone.dashboard.repository.GitHubRepoRepository;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.validation.annotation.Validated;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+import javax.validation.Valid;
+import java.util.List;
+import java.util.Objects;
+
+import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE;
+import static org.springframework.web.bind.annotation.RequestMethod.GET;
+
+@RestController
+@Validated
+@RequestMapping("/github")
+public class GitHubController {
+    private static final Log LOG = LogFactory.getLog(GitHubController.class);
+
+    private final BaseCollectorRepository<Collector> collectorRepository;
+    private final GitHubRepoRepository gitHubRepoRepository;
+    private final GitHubCollectorTask gitHubCollectorTask;
+    private static final String GITHUB_COLLECTOR_NAME = "GitHub";
+
+    @Autowired
+    public GitHubController(BaseCollectorRepository<Collector> collectorRepository,
+                            GitHubRepoRepository gitHubRepoRepository,
+                            GitHubCollectorTask gitHubCollectorTask) {
+        this.collectorRepository = collectorRepository;
+        this.gitHubRepoRepository = gitHubRepoRepository;
+        this.gitHubCollectorTask = gitHubCollectorTask;
+    }
+
+    @RequestMapping(value = "/refresh", method = GET, produces = APPLICATION_JSON_VALUE)
+    public ResponseEntity<String> refresh(@Valid String url) throws HygieiaException {
+        if (Objects.isNull(url)) return ResponseEntity.status(HttpStatus.OK).body("URL is null");
+        Collector collector = collectorRepository.findByName(GITHUB_COLLECTOR_NAME);
+        if (Objects.isNull(collector))
+            return ResponseEntity.status(HttpStatus.OK).body(GITHUB_COLLECTOR_NAME + " collector is not found");
+        List<GitHubRepo> repos = gitHubRepoRepository.findByCollectorIdAndUrl(collector.getId(), url);
+        if (CollectionUtils.isEmpty(repos))
+            return ResponseEntity.status(HttpStatus.OK).body("No GitHub repos found for URL: " + url);
+        int count = 0;
+        gitHubCollectorTask.collectProcess(collector, repos);
+
+        return ResponseEntity
+                .status(HttpStatus.OK)
+                .body(GITHUB_COLLECTOR_NAME + " refresh completed successfully for repo URL: " + url
+                        + " Records updated: " + collector.getLastExecutionRecordCount());
+    }
+
+}

src/main/java/com/capitalone/dashboard/controller/GitHubController.java

@@ -10,4 +10,7 @@ public interface GitHubRepoRepository extends BaseCollectorItemRepository<GitHub
 
     @Query(value="{ 'collectorId' : ?0, enabled: true}")
     List<GitHubRepo> findEnabledGitHubRepos(ObjectId collectorId);
+
+    @Query(value="{ 'collectorId' : ?0, options.url : ?1}")
+    List<GitHubRepo> findByCollectorIdAndUrl(ObjectId collectorId, String url);
 }

src/main/java/com/capitalone/dashboard/repository/GitHubRepoRepository.java

@@ -1,11 +1,11 @@
 package com.capitalone.dashboard.collector;
 
 import com.capitalone.dashboard.client.RestClient;
+import com.capitalone.dashboard.client.RestOperationsSupplier;
 import com.capitalone.dashboard.misc.HygieiaException;
 import com.capitalone.dashboard.model.GitHubRepo;
 import com.capitalone.dashboard.model.GitHubParsed;
 import com.capitalone.dashboard.collector.DefaultGitHubClient.RedirectedStatus;
-import com.capitalone.dashboard.util.Supplier;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -27,7 +27,7 @@
 public class DefaultGitHubClientTest {
 
     @Mock
-    private Supplier<RestOperations> restOperationsSupplier;
+    private RestOperationsSupplier restOperationsSupplier;
     @Mock private RestOperations rest;
     private GitHubSettings settings;
     private DefaultGitHubClient defaultGitHubClient;