concord-server: remove GithubTriggerProcessor interface (#974)

Author: ibodrov

server/impl/src/main/java/com/walmartlabs/concord/server/events/EventModule.java

@@ -22,14 +22,17 @@
 
 import com.google.inject.Binder;
 import com.google.inject.Module;
+import com.walmartlabs.concord.server.events.github.GithubTriggerProcessor;
 import com.walmartlabs.concord.server.sdk.events.ProcessEventListener;
 
+import static com.google.inject.Scopes.SINGLETON;
 import static com.google.inject.multibindings.Multibinder.newSetBinder;
 
 public class EventModule implements Module {
 
     @Override
     public void configure(Binder binder) {
         newSetBinder(binder, ProcessEventListener.class);
+        binder.bind(GithubTriggerProcessor.class).in(SINGLETON);
     }
 }

server/impl/src/main/java/com/walmartlabs/concord/server/events/GithubEventResource.java

@@ -86,7 +86,7 @@ public class GithubEventResource implements Resource {
     private final GithubConfiguration githubCfg;
     private final TriggerProcessExecutor executor;
     private final AuditLog auditLog;
-    private final List<GithubTriggerProcessor> processors;
+    private final GithubTriggerProcessor processor;
     private final UserManager userManager;
     private final LdapManager ldapManager;
     private final TriggerEventInitiatorResolver initiatorResolver;
@@ -96,7 +96,7 @@ public class GithubEventResource implements Resource {
     public GithubEventResource(GithubConfiguration githubCfg,
                                TriggerProcessExecutor executor,
                                AuditLog auditLog,
-                               List<GithubTriggerProcessor> processors,
+                               GithubTriggerProcessor processor,
                                UserManager userManager,
                                LdapManager ldapManager,
                                TriggerEventInitiatorResolver initiatorResolver,
@@ -105,7 +105,7 @@ public GithubEventResource(GithubConfiguration githubCfg,
         this.githubCfg = githubCfg;
         this.executor = executor;
         this.auditLog = auditLog;
-        this.processors = processors;
+        this.processor = processor;
         this.userManager = userManager;
         this.ldapManager = ldapManager;
         this.initiatorResolver = initiatorResolver;
@@ -153,7 +153,7 @@ public String onEvent(Map<String, Object> data,
         }
 
         List<GithubTriggerProcessor.Result> results = new ArrayList<>();
-        processors.forEach(p -> p.process(eventName, payload, uriInfo, results));
+        processor.process(eventName, payload, uriInfo, results);
 
         Supplier<UserEntry> initiatorSupplier = memo(new GithubEventInitiatorSupplier(userManager, ldapManager, payload));
 
@@ -202,7 +202,7 @@ public Map<String, Object> resolve(TriggerEntry t) {
 
             GithubTriggerExclusiveMode e = objectMapper.convertValue(exclusive, GithubTriggerExclusiveMode.class);
             String groupBy = e.groupByProperty();
-            if (groupBy== null) {
+            if (groupBy == null) {
                 return exclusive;
             }
 

server/impl/src/main/java/com/walmartlabs/concord/server/events/github/GithubTriggerProcessor.java

@@ -11,7 +11,7 @@
  * You may obtain a copy of the License at
  * 
  *      http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *   
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -20,42 +20,291 @@
  * =====
  */
 
+import com.walmartlabs.concord.db.MainDB;
+import com.walmartlabs.concord.sdk.Constants.Trigger;
+import com.walmartlabs.concord.sdk.MapUtils;
+import com.walmartlabs.concord.server.cfg.GithubConfiguration;
+import com.walmartlabs.concord.server.events.DefaultEventFilter;
+import com.walmartlabs.concord.server.org.project.RepositoryDao;
+import com.walmartlabs.concord.server.org.project.RepositoryEntry;
 import com.walmartlabs.concord.server.org.triggers.TriggerEntry;
+import com.walmartlabs.concord.server.org.triggers.TriggerUtils;
+import com.walmartlabs.concord.server.org.triggers.TriggersDao;
+import com.walmartlabs.concord.server.sdk.metrics.WithTimer;
+import com.walmartlabs.concord.server.security.github.GithubKey;
+import org.jooq.Configuration;
+import org.jooq.DSLContext;
+import org.jooq.impl.DSL;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
+import javax.inject.Inject;
+import javax.inject.Named;
+import javax.inject.Singleton;
 import javax.ws.rs.core.UriInfo;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+
+import static com.walmartlabs.concord.server.events.github.Constants.*;
+
+public class GithubTriggerProcessor {
+
+    private static final int VERSION_ID = 2;
+    private static final Logger log = LoggerFactory.getLogger(GithubTriggerProcessor.class);
+
+    private final Dao dao;
+    private final List<EventEnricher> eventEnrichers;
+    private final boolean isDisableReposOnDeletedRef;
+
+    @Inject
+    public GithubTriggerProcessor(Dao dao,
+                                  List<EventEnricher> eventEnrichers,
+                                  GithubConfiguration githubCfg) {
+        this.dao = dao;
+        this.eventEnrichers = eventEnrichers;
+        this.isDisableReposOnDeletedRef = githubCfg.isDisableReposOnDeletedRef();
+    }
+
+    @WithTimer
+    public void process(String eventName, Payload payload, UriInfo uriInfo, List<Result> result) {
+        GithubKey githubKey = GithubKey.getCurrent();
+        UUID projectId = githubKey.getProjectId();
+
+        if (isDisableReposOnDeletedRef
+            && PUSH_EVENT.equals(payload.eventName())
+            && isRefDeleted(payload)) {
+
+            List<RepositoryEntry> repositories = dao.findRepos(payload.getFullRepoName());
+            // disable repos configured with the event branch
+            repositories.stream()
+                    .filter(r -> !r.isDisabled() && null != r.getBranch())
+                    .filter(r -> r.getBranch().equals(payload.getBranch()))
+                    .forEach(r -> disableRepo(r, payload));
+        }
+
+        List<TriggerEntry> triggers = listTriggers(projectId, payload.getOrg(), payload.getRepo());
+        for (TriggerEntry t : triggers) {
+            if (skipTrigger(t, eventName, payload)) {
+                continue;
+            }
+
+            Map<String, Object> event = buildEvent(eventName, uriInfo, payload);
+            enrichEventConditions(payload, t, event);
+
+            if (DefaultEventFilter.filter(event, t)) {
+                result.add(new Result(event, t));
+            }
+        }
+    }
+
+    static boolean skipTrigger(TriggerEntry t, String eventName, Payload payload) {
+        // skip empty push events if the trigger's configuration says so
+        if (GithubUtils.ignoreEmptyPush(t) && GithubUtils.isEmptyPush(eventName, payload)) {
+            return true;
+        }
+
+        // process is destined to fail if attempted to start from commit in another repo
+        // on an event from a pull request.
+        if (TriggerUtils.isUseEventCommitId(t)
+            && payload.hasPullRequestEntry()
+            && payload.isPullRequestFromDifferentRepo()) {
+
+            log.info("Skip start from {} event [{}, {}] -> Commit is in a different repository.",
+                    eventName, payload.getPullRequestBaseUrl(), payload.getPullRequestHeadUrl());
+
+            return true;
+        }
+
+        return false;
+    }
+
+    private void enrichEventConditions(Payload payload, TriggerEntry trigger, Map<String, Object> result) {
+        for (EventEnricher e : eventEnrichers) {
+            e.enrich(payload, trigger, result);
+        }
+    }
+
+    private void disableRepo(RepositoryEntry repo, Payload payload) {
+        log.info("disable repo ['{}', '{}'] -> ref deleted", repo.getId(), payload.getBranch());
+        dao.disable(repo.getProjectId(), repo.getId());
+    }
+
+    private static boolean isRefDeleted(Payload payload) {
+        Object val = payload.raw().get("deleted");
+
+        if (val == null) {
+            return false;
+        }
+
+        if (val instanceof String str) {
+            return Boolean.parseBoolean(str);
+        }
 
-public interface GithubTriggerProcessor {
+        return Boolean.TRUE.equals(val);
+    }
+
+    @WithTimer
+    List<TriggerEntry> listTriggers(UUID projectId, String org, String repo) {
+        return dao.listTriggers(projectId, org, repo);
+    }
+
+    private Map<String, Object> buildEvent(String eventName, UriInfo uriInfo, Payload payload) {
+        Map<String, Object> result = new HashMap<>();
+
+        result.put(GITHUB_ORG_KEY, payload.getOrg());
+        result.put(GITHUB_REPO_KEY, payload.getRepo());
+        result.put(GITHUB_HOST_KEY, payload.getHost());
+        String branch = payload.getBranch();
+        if (branch != null) {
+            result.put(REPO_BRANCH_KEY, payload.getBranch());
+        }
+
+        if (PULL_REQUEST_EVENT.equals(eventName)) {
+            Map<String, Object> pullRequest = MapUtils.getMap(payload.raw(), PULL_REQUEST_EVENT, Collections.emptyMap());
+            Map<String, Object> head = MapUtils.getMap(pullRequest, "head", Collections.emptyMap());
+            String sha = MapUtils.getString(head, "sha");
+            if (sha != null) {
+                result.put(COMMIT_ID_KEY, sha);
+            }
+        } else if (PUSH_EVENT.equals(eventName)) {
+            String after = payload.getString("after");
+            if (after != null) {
+                result.put(COMMIT_ID_KEY, after);
+            }
+        }
+
+        result.put(SENDER_KEY, payload.getSender());
+        result.put(TYPE_KEY, eventName);
+        result.put(STATUS_KEY, payload.getAction());
+        result.put(PAYLOAD_KEY, payload.raw());
+        result.put(QUERY_PARAMS_KEY, new HashMap<>(uriInfo.getQueryParameters()));
+
+        // files
+        Map<String, Set<String>> files = new HashMap<>(payload.getFiles());
+        // alias for all files (changed/modified/deleted)
+        files.put("any", files.values().stream()
+                .flatMap(Set::stream)
+                .collect(Collectors.toSet()));
+        result.put(FILES_KEY, files);
+
+        // match only with v2 triggers
+        result.put(VERSION_KEY, VERSION_ID);
+
+        return result;
+    }
+
+    public interface EventEnricher {
+
+        void enrich(Payload payload, TriggerEntry trigger, Map<String, Object> result);
+    }
+
+    /**
+     * Adds {@link Trigger#REPOSITORY_INFO} property to the event, but only if
+     * the trigger's conditions contained the clause with the same key.
+     */
+    @Named
+    private static class RepositoryInfoEnricher implements EventEnricher {
 
-    void process(String eventName, Payload payload, UriInfo uriInfo, List<Result> result);
+        private final Dao dao;
 
-    class Result {
+        @Inject
+        public RepositoryInfoEnricher(Dao dao) {
+            this.dao = dao;
+        }
+
+        @Override
+        @WithTimer
+        public void enrich(Payload payload, TriggerEntry trigger, Map<String, Object> result) {
+            Object projectInfoConditions = trigger.getConditions().get(Trigger.REPOSITORY_INFO);
+            if (projectInfoConditions == null || payload.getFullRepoName() == null) {
+                return;
+            }
+
+            List<Map<String, Object>> repositoryInfos = new ArrayList<>();
+            List<RepositoryEntry> repositories = dao.findRepos(payload.getFullRepoName());
 
-        private final Map<String, Object> event;
+            for (RepositoryEntry r : repositories) {
+                if (r.isDisabled()) {
+                    continue;
+                }
 
-        private final List<TriggerEntry> triggers;
+                Map<String, Object> repositoryInfo = new HashMap<>();
+                repositoryInfo.put(REPO_ID_KEY, r.getId());
+                repositoryInfo.put(REPO_NAME_KEY, r.getName());
+                repositoryInfo.put(PROJECT_ID_KEY, r.getProjectId());
+                if (r.getBranch() != null) {
+                    repositoryInfo.put(REPO_BRANCH_KEY, r.getBranch());
+                }
+                repositoryInfo.put(REPO_ENABLED_KEY, !r.isDisabled());
 
-        public Result(Map<String, Object> event, List<TriggerEntry> triggers) {
-            this.event = event;
-            this.triggers = triggers;
+                repositoryInfos.add(repositoryInfo);
+            }
+
+            if (!repositoryInfos.isEmpty()) {
+                result.put(Trigger.REPOSITORY_INFO, repositoryInfos);
+            }
         }
+    }
 
-        public Map<String, Object> event() {
-            return event;
+    @Named
+    @Singleton
+    public static class Dao {
+        private final RepositoryDao repoDao;
+        private final TriggersDao triggersDao;
+        private final Configuration cfg;
+
+        @Inject
+        public Dao(@MainDB Configuration cfg,
+                   RepositoryDao repoDao,
+                   TriggersDao triggersDao) {
+            this.cfg = cfg;
+            this.triggersDao = triggersDao;
+            this.repoDao = repoDao;
         }
 
-        public List<TriggerEntry> triggers() {
-            return triggers;
+        private List<RepositoryEntry> findRepos(String repoOrgAndName) {
+            String sshAndHttpPattern = "%[/:]" + repoOrgAndName + "(.git)?/?";
+            return repoDao.findSimilar(sshAndHttpPattern);
         }
 
-        static Result from(Map<String, Object> event, TriggerEntry trigger) {
-            return new Result(event, Collections.singletonList(trigger));
+        List<TriggerEntry> listTriggers(UUID projectId, String org, String repo) {
+            Map<String, String> conditions = new HashMap<>();
+
+            if (org != null) {
+                conditions.put(GITHUB_ORG_KEY, org);
+            }
+
+            if (repo != null) {
+                conditions.put(GITHUB_REPO_KEY, repo);
+            }
+
+            return triggersDao.list(projectId, EVENT_SOURCE, VERSION_ID, conditions);
         }
 
-        static Result from(Map<String, Object> event, List<TriggerEntry> triggers) {
-            return new Result(event, triggers);
+        void disable(UUID projectId, UUID repoId) {
+            tx(tx -> {
+                repoDao.disable(tx, repoId);
+                triggersDao.delete(tx, projectId, repoId);
+            });
+        }
+
+        private DSLContext dsl() {
+            return DSL.using(cfg);
+        }
+
+        private void tx(Consumer<DSLContext> c) {
+            dsl().transaction(localCfg -> {
+                DSLContext tx = DSL.using(localCfg);
+                c.accept(tx);
+            });
+        }
+    }
+
+    public record Result(Map<String, Object> event, List<TriggerEntry> triggers) {
+
+        private Result(Map<String, Object> event, TriggerEntry trigger) {
+            this(event, List.of(trigger));
         }
     }
 }