Merge branch 'master' into update-dependencies

Author: aochsner

src/main/java/com/capitalone/dashboard/collector/GitHubCollectorTask.java

@@ -171,25 +171,25 @@ public void collect(Collector collector){
                enabledRepos = enabledRepos.stream().filter(repo -> GithubRepoMatcher.orgNameMatcher(repo.getRepoUrl(), searchCriteria[1])).collect(Collectors.toList());
            }
        }
+       LOG.info("GitHubCollectorTask:collect start, total enabledRepos=" + enabledRepos.size());
+       LOG.warn("error threshold = " + gitHubSettings.getErrorThreshold());
        collectProcess(collector, enabledRepos );
+
    }
 
 
     @SuppressWarnings({"PMD.AvoidDeeplyNestedIfStmts"})
     public void collectProcess(Collector collector, List<GitHubRepo> enabledRepos) {
-
-        logBanner("Starting...");
         long start = System.currentTimeMillis();
         int repoCount = 0;
         int commitCount = 0;
         int pullCount = 0;
         int issueCount = 0;
         count.set(0);
 
-        LOG.info("GitHubCollectorTask:collect start, total enabledRepos=" + enabledRepos.size());
-        LOG.warn("error threshold = " + gitHubSettings.getErrorThreshold());
         for (GitHubRepo repo : enabledRepos) {
             repoCount++;
+            long repoStart = System.currentTimeMillis();
             String repoUrl = repo==null?"null":(repo.getRepoUrl() + "/tree/" + repo.getBranch());
             String statusString = "UNKNOWN";
             long lastUpdated = repo==null?0:repo.getLastUpdated();
@@ -241,7 +241,7 @@ public void collectProcess(Collector collector, List<GitHubRepo> enabledRepos) {
                         repo.setLastUpdated(System.currentTimeMillis());
                         // if everything went alright, there should be no error!
                         repo.getErrors().clear();
-                        statusString = "SUCCESS, pulls=" + pullCount1 + ", commits=" + pullCount1 + ", issues=" + issueCount1;
+                        statusString = "SUCCESS, pulls=" + pullCount1 + ", commits=" + commitCount1 + ", issues=" + issueCount1;
                     } catch (HttpStatusCodeException hc) {
                         LOG.error("Error fetching commits for:" + repo.getRepoUrl(), hc);
                         statusString = "EXCEPTION, " + hc.getClass().getCanonicalName();
@@ -275,16 +275,19 @@ public void collectProcess(Collector collector, List<GitHubRepo> enabledRepos) {
                 LOG.error("Unexpected exception when collecting url=" + repoUrl, e);
             } finally {
                 String age = readableAge(lastUpdated, start);
-                LOG.info(String.format("%d of %d, repository=%s, lastUpdated=%d [%s], status=%s",
-                        repoCount, enabledRepos.size(), repoUrl, lastUpdated, age, statusString));
-
+                long repoEnd = System.currentTimeMillis();
+                long repoProcSeconds = (repoEnd - repoStart) / 1000;
+                LOG.info(String.format("%d of %d, repository=%s, itemProcessSeconds=%d, lastUpdated=%d [%s], status=%s",
+                        repoCount, enabledRepos.size(), repoUrl, repoProcSeconds, lastUpdated, age, statusString));
             }
         }
         long end = System.currentTimeMillis();
         long elapsedSeconds = (end - start) / 1000;
         count.set(commitCount);
         LOG.info(String.format("GitHubCollectorTask:collect stop, totalProcessSeconds=%d, totalRepoCount=%d, totalNewPulls=%d, totalNewCommits=%d totalNewIssues=%d",
                 elapsedSeconds, repoCount, pullCount, commitCount, issueCount));
+
+        collector.setLastExecutionRecordCount(repoCount+pullCount+commitCount+issueCount);
     }
 
 

src/main/java/com/capitalone/dashboard/controller/GitHubController.java

@@ -0,0 +1,63 @@
+package com.capitalone.dashboard.controller;
+
+import com.capitalone.dashboard.collector.GitHubCollectorTask;
+import com.capitalone.dashboard.misc.HygieiaException;
+import com.capitalone.dashboard.model.Collector;
+import com.capitalone.dashboard.model.GitHubRepo;
+import com.capitalone.dashboard.repository.BaseCollectorRepository;
+import com.capitalone.dashboard.repository.GitHubRepoRepository;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.validation.annotation.Validated;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+import javax.validation.Valid;
+import java.util.List;
+import java.util.Objects;
+
+import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE;
+import static org.springframework.web.bind.annotation.RequestMethod.GET;
+
+@RestController
+@Validated
+public class GitHubController {
+    private static final Log LOG = LogFactory.getLog(GitHubController.class);
+
+    private final BaseCollectorRepository<Collector> collectorRepository;
+    private final GitHubRepoRepository gitHubRepoRepository;
+    private final GitHubCollectorTask gitHubCollectorTask;
+    private static final String GITHUB_COLLECTOR_NAME = "GitHub";
+
+    @Autowired
+    public GitHubController(BaseCollectorRepository<Collector> collectorRepository,
+                            GitHubRepoRepository gitHubRepoRepository,
+                            GitHubCollectorTask gitHubCollectorTask) {
+        this.collectorRepository = collectorRepository;
+        this.gitHubRepoRepository = gitHubRepoRepository;
+        this.gitHubCollectorTask = gitHubCollectorTask;
+    }
+
+    @RequestMapping(value = "/refresh", method = GET, produces = APPLICATION_JSON_VALUE)
+    public ResponseEntity<String> refresh(@Valid String url) throws HygieiaException {
+        if (Objects.isNull(url)) return ResponseEntity.status(HttpStatus.OK).body("URL is null");
+        Collector collector = collectorRepository.findByName(GITHUB_COLLECTOR_NAME);
+        if (Objects.isNull(collector))
+            return ResponseEntity.status(HttpStatus.OK).body(GITHUB_COLLECTOR_NAME + " collector is not found");
+        List<GitHubRepo> repos = gitHubRepoRepository.findByCollectorIdAndUrl(collector.getId(), url);
+        if (CollectionUtils.isEmpty(repos))
+            return ResponseEntity.status(HttpStatus.OK).body("No GitHub repos found for URL: " + url);
+        int count = 0;
+        gitHubCollectorTask.collectProcess(collector, repos);
+
+        return ResponseEntity
+                .status(HttpStatus.OK)
+                .body(GITHUB_COLLECTOR_NAME + " refresh completed successfully for repo URL: " + url
+                        + " Records updated: " + collector.getLastExecutionRecordCount());
+    }
+
+}

src/main/java/com/capitalone/dashboard/repository/GitHubRepoRepository.java

@@ -10,4 +10,7 @@ public interface GitHubRepoRepository extends BaseCollectorItemRepository<GitHub
 
     @Query(value="{ 'collectorId' : ?0, enabled: true}")
     List<GitHubRepo> findEnabledGitHubRepos(ObjectId collectorId);
+
+    @Query(value="{ 'collectorId' : ?0, options.url : ?1}")
+    List<GitHubRepo> findByCollectorIdAndUrl(ObjectId collectorId, String url);
 }