server, agent: GitHub app installation support for repository cloning

agent/pom.xml

@@ -29,6 +29,10 @@
             <groupId>com.walmartlabs.concord</groupId>
             <artifactId>concord-common</artifactId>
         </dependency>
+        <dependency>
+            <groupId>com.walmartlabs.concord</groupId>
+            <artifactId>concord-github-app-installation</artifactId>
+        </dependency>
         <dependency>
             <groupId>com.walmartlabs.concord</groupId>
             <artifactId>concord-client2</artifactId>
@@ -143,6 +147,11 @@
             <artifactId>mockito-core</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-junit-jupiter</artifactId>
+            <scope>test</scope>
+        </dependency>
 
         <!-- dependency tree fix for mvnd -->
         <dependency>

agent/src/main/java/com/walmartlabs/concord/agent/AgentAuthTokenProvider.java

@@ -0,0 +1,135 @@
+package com.walmartlabs.concord.agent;
+
+/*-
+ * *****
+ * Concord
+ * -----
+ * Copyright (C) 2017 - 2025 Walmart Inc.
+ * -----
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =====
+ */
+
+import com.typesafe.config.Config;
+import com.walmartlabs.concord.agent.remote.ApiClientFactory;
+import com.walmartlabs.concord.client2.ApiException;
+import com.walmartlabs.concord.client2.SystemApi;
+import com.walmartlabs.concord.common.AuthTokenProvider;
+import com.walmartlabs.concord.common.ExternalAuthToken;
+import com.walmartlabs.concord.common.cfg.MappingAuthConfig;
+import com.walmartlabs.concord.github.appinstallation.GitHubAppInstallation;
+import com.walmartlabs.concord.sdk.Secret;
+
+import javax.annotation.Nullable;
+import javax.inject.Inject;
+import java.io.IOException;
+import java.net.URI;
+import java.util.List;
+import java.util.Optional;
+
+public class AgentAuthTokenProvider implements AuthTokenProvider {
+
+    private final List<AuthTokenProvider> authTokenProviders;
+
+    @Inject
+    public AgentAuthTokenProvider(ConcordServerTokenProvider concordProvider,
+                                  GitHubAppInstallation githubProvider,
+                                  AuthTokenProvider.OauthTokenProvider oauthTokenProvider) {
+
+        this.authTokenProviders = List.of(
+                concordProvider,
+                githubProvider,
+                oauthTokenProvider
+        );
+    }
+
+    @Override
+    public boolean supports(URI repo, @Nullable Secret secret) {
+        return authTokenProviders.stream()
+                .anyMatch(p -> p.supports(repo, secret));
+    }
+
+    public Optional<ExternalAuthToken> getToken(URI repo, @Nullable Secret secret) {
+        for (var k : authTokenProviders) {
+            if (k.supports(repo, secret)) {
+                return k.getToken(repo, secret);
+            }
+        }
+
+        return Optional.empty();
+    }
+
+    public static class ConcordServerTokenProvider implements AuthTokenProvider {
+
+        private static final String CFG_ENABLED = "externalTokenProvider.enabled";
+        private static final String CFG_URL_PATTERN = "externalTokenProvider.urlPattern";
+
+        private final SystemApi systemApi;
+        private final MappingAuthConfig.ConcordServerAuthConfig auth;
+
+        @Inject
+        public ConcordServerTokenProvider(ApiClientFactory apiClientFactory, Config config) {
+            this.auth = initAuth(config);
+
+            try {
+                this.systemApi = new SystemApi(apiClientFactory.create(null));
+            } catch (IOException e) {
+                throw new RuntimeException("Error initializing System API client", e);
+            }
+        }
+
+        private static MappingAuthConfig.ConcordServerAuthConfig initAuth(Config config) {
+            if (!config.hasPath(CFG_ENABLED) || !config.getBoolean(CFG_ENABLED)) {
+                return null;
+            }
+
+            return config.hasPath(CFG_URL_PATTERN)
+                    ? MappingAuthConfig.ConcordServerAuthConfig.builder()
+                            .urlPattern(MappingAuthConfig.assertBaseUrlPattern(config.getString(CFG_URL_PATTERN)))
+                            .build()
+                    : null;
+        }
+
+        @Override
+        public boolean supports(URI repo, @Nullable Secret secret) {
+            if (auth == null) {
+                return false;
+            }
+
+            // Maybe support secret input? This requires elevated api access permission
+
+            return auth.canHandle(repo);
+        }
+
+        @Override
+        public Optional<ExternalAuthToken> getToken(URI repo, @Nullable Secret secret) {
+            try {
+                var resp = systemApi.getExternalToken(repo);
+
+                return Optional.of(ExternalAuthToken.StaticToken.builder()
+                        .token(resp.getToken())
+                        .username(resp.getUsername())
+                        .expiresAt(resp.getExpiresAt())
+                        .build());
+            } catch (ApiException e) {
+                if (e.getCode() == 403) {
+                    // User needs externalTokenLookup permission
+                    throw new RuntimeException("No permission to get auth token from concord server.");
+                }
+
+                throw new RuntimeException("Error retrieving concord-provided auth token", e);
+            }
+        }
+    }
+
+}

agent/src/main/java/com/walmartlabs/concord/agent/AgentModule.java

@@ -30,7 +30,9 @@
 import com.walmartlabs.concord.agent.remote.ApiClientFactory;
 import com.walmartlabs.concord.agent.remote.QueueClientProvider;
 import com.walmartlabs.concord.common.ObjectMapperProvider;
+import com.walmartlabs.concord.common.cfg.OauthTokenConfig;
 import com.walmartlabs.concord.config.ConfigModule;
+import com.walmartlabs.concord.github.appinstallation.cfg.GitHubAppInstallationConfig;
 import com.walmartlabs.concord.server.queueclient.QueueClient;
 
 import javax.inject.Named;
@@ -59,6 +61,10 @@ public void configure(Binder binder) {
         binder.bind(DockerConfiguration.class).in(SINGLETON);
         binder.bind(RuntimeConfiguration.class).asEagerSingleton();
         binder.bind(GitConfiguration.class).in(SINGLETON);
+        binder.bind(OauthTokenConfig.class).to(GitConfiguration.class).in(SINGLETON);
+        binder.bind(GitHubConfiguration.class).in(SINGLETON);
+        binder.bind(GitHubAppInstallationConfig.class).to(GitHubConfiguration.class).in(SINGLETON);
+        binder.bind(AgentAuthTokenProvider.ConcordServerTokenProvider.class).in(SINGLETON);
         binder.bind(ImportConfiguration.class).in(SINGLETON);
         binder.bind(PreForkConfiguration.class).in(SINGLETON);
         binder.bind(RepositoryCacheConfiguration.class).in(SINGLETON);

agent/src/main/java/com/walmartlabs/concord/agent/RepositoryManager.java

@@ -34,7 +34,6 @@
 import javax.inject.Inject;
 import java.io.IOException;
 import java.nio.file.Path;
-import java.util.Arrays;
 import java.util.List;
 
 public class RepositoryManager {
@@ -46,29 +45,33 @@ public class RepositoryManager {
     private final RepositoryCache repositoryCache;
     private final GitConfiguration gitCfg;
 
+
     @Inject
     public RepositoryManager(SecretClient secretClient,
                              GitConfiguration gitCfg,
                              RepositoryCacheConfiguration cacheCfg,
                              ObjectMapper objectMapper,
-                             DependencyManager dependencyManager) throws IOException {
+                             DependencyManager dependencyManager,
+                             AgentAuthTokenProvider agentAuthTokenProvider) throws IOException {
 
         this.secretClient = secretClient;
         this.gitCfg = gitCfg;
 
         GitClientConfiguration clientCfg = GitClientConfiguration.builder()
-                .oauthToken(gitCfg.getToken())
+                .oauthToken(gitCfg.getOauthToken())
                 .defaultOperationTimeout(gitCfg.getDefaultOperationTimeout())
                 .fetchTimeout(gitCfg.getFetchTimeout())
                 .httpLowSpeedLimit(gitCfg.getHttpLowSpeedLimit())
                 .httpLowSpeedTime(gitCfg.getHttpLowSpeedTime())
+                .allowedSchemes(gitCfg.getAllowedSchemes())
                 .sshTimeout(gitCfg.getSshTimeout())
                 .sshTimeoutRetryCount(gitCfg.getSshTimeoutRetryCount())
                 .build();
 
-        List<RepositoryProvider> providers = Arrays.asList(new MavenRepositoryProvider(dependencyManager), new GitCliRepositoryProvider(clientCfg));
-        this.providers = new RepositoryProviders(providers);
-
+        this.providers = new RepositoryProviders(List.of(
+                new MavenRepositoryProvider(dependencyManager),
+                new GitCliRepositoryProvider(clientCfg, agentAuthTokenProvider)
+        ));
         this.repositoryCache = new RepositoryCache(cacheCfg.getCacheDir(),
                 cacheCfg.getInfoDir(),
                 cacheCfg.getLockTimeout(),

agent/src/main/java/com/walmartlabs/concord/agent/cfg/GitConfiguration.java

@@ -21,15 +21,21 @@
  */
 
 import com.typesafe.config.Config;
+import com.walmartlabs.concord.common.cfg.MappingAuthConfig;
+import com.walmartlabs.concord.common.cfg.OauthTokenConfig;
 
 import javax.inject.Inject;
 import java.time.Duration;
+import java.util.List;
+import java.util.Optional;
 
 import static com.walmartlabs.concord.agent.cfg.Utils.getStringOrDefault;
 
-public class GitConfiguration {
+public class GitConfiguration implements OauthTokenConfig {
 
     private final String token;
+    private final String oauthUsername;
+    private final String oauthUrlPattern;
     private final boolean shallowClone;
     private final boolean checkAlreadyFetched;
     private final Duration defaultOperationTimeout;
@@ -39,10 +45,14 @@ public class GitConfiguration {
     private final Duration sshTimeout;
     private final int sshTimeoutRetryCount;
     private final boolean skip;
+    private final List<String> allowedSchemes;
+    private final List<? extends Config> authConfigs;
 
     @Inject
     public GitConfiguration(Config cfg) {
         this.token = getStringOrDefault(cfg, "git.oauth", () -> null);
+        this.oauthUsername = getStringOrDefault(cfg, "git.oauthUsername", () -> null);
+        this.oauthUrlPattern = getStringOrDefault(cfg, "git.oauthUrlPattern", () -> null);
         this.shallowClone = cfg.getBoolean("git.shallowClone");
         this.checkAlreadyFetched = cfg.getBoolean("git.checkAlreadyFetched");
         this.defaultOperationTimeout = cfg.getDuration("git.defaultOperationTimeout");
@@ -52,10 +62,23 @@ public GitConfiguration(Config cfg) {
         this.sshTimeout = cfg.getDuration("git.sshTimeout");
         this.sshTimeoutRetryCount = cfg.getInt("git.sshTimeoutRetryCount");
         this.skip = cfg.getBoolean("git.skip");
+        this.allowedSchemes = cfg.getStringList("git.allowedSchemes");
+        this.authConfigs = cfg.getConfigList("git.systemAuth");
     }
 
-    public String getToken() {
-        return token;
+    @Override
+    public Optional<String> getOauthToken() {
+        return Optional.ofNullable(token);
+    }
+
+    @Override
+    public Optional<String> getOauthUsername() {
+        return Optional.ofNullable(oauthUsername);
+    }
+
+    @Override
+    public Optional<String> getOauthUrlPattern() {
+        return Optional.ofNullable(oauthUrlPattern);
     }
 
     public boolean isShallowClone() {
@@ -93,4 +116,47 @@ public int getSshTimeoutRetryCount() {
     public boolean isSkip() {
         return skip;
     }
+
+    public List<String> getAllowedSchemes() { return allowedSchemes; }
+
+    public List<MappingAuthConfig> getSystemAuth() {
+        return authConfigs.stream()
+                .map(o -> {
+                    AuthSource type = AuthSource.valueOf(o.getString("type").toUpperCase());
+
+                    return (AuthConfig) switch (type) {
+                        case OAUTH_TOKEN -> OauthConfig.from(o);
+                    };
+                })
+                .map(AuthConfig::toGitAuth)
+                .toList();
+
+    }
+
+    enum AuthSource {
+        OAUTH_TOKEN
+    }
+
+    public interface AuthConfig {
+        MappingAuthConfig toGitAuth();
+    }
+
+    public record OauthConfig(String urlPattern, String token) implements AuthConfig {
+
+        static OauthConfig from(Config cfg) {
+            return new OauthConfig(
+                    cfg.getString("urlPattern"),
+                    cfg.getString("token")
+            );
+        }
+
+        @Override
+        public MappingAuthConfig.OauthAuthConfig toGitAuth() {
+            return MappingAuthConfig.OauthAuthConfig.builder()
+                    .urlPattern(MappingAuthConfig.assertBaseUrlPattern(this.urlPattern()))
+                    .token(this.token())
+                    .build();
+        }
+    }
+
 }