Skip to content

Commit b34d525

Browse files
caioqcaioq
authored
Gitextractor auth cherry (#8182)
* feat: generating new github access token to every gitextractor task Signed-off-by: Caio Queiroz <caiogqueiroz@gmail.com> * feat: using DynamicGitUrl interface to implement the git url logic * refactor: remove unused code * fix: unit test * fix: lint --------- Signed-off-by: Caio Queiroz <caiogqueiroz@gmail.com>
1 parent f9dfac1 commit b34d525

File tree

6 files changed

+103
-11
lines changed

6 files changed

+103
-11
lines changed

‎backend/plugins/gitextractor/impl/impl.go‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ limitations under the License.
1818
package impl
1919

2020
import (
21+
"fmt"
2122
"net/url"
2223

2324
"github.com/apache/incubator-devlake/core/dal"
@@ -37,6 +38,10 @@ var _ interface {
3738

3839
type GitExtractor struct{}
3940

41+
type DynamicGitUrl interface {
42+
GetDynamicGitUrl(taskCtx plugin.TaskContext, connectionId uint64, repoUrl string) (string, errors.Error)
43+
}
44+
4045
func (p GitExtractor) GetTablesInfo() []dal.Tabler {
4146
return []dal.Tabler{}
4247
}
@@ -68,6 +73,24 @@ func (p GitExtractor) PrepareTaskData(taskCtx plugin.TaskContext, options map[st
6873
return nil, err
6974
}
7075

76+
if op.PluginName != "" {
77+
pluginInstance, err := plugin.GetPlugin(op.PluginName)
78+
if err != nil {
79+
return nil, errors.Default.Wrap(err, fmt.Sprintf("failed to get plugin instance for plugin: %s", op.PluginName))
80+
}
81+
82+
if pluginGit, ok := pluginInstance.(DynamicGitUrl); ok {
83+
gitUrl, err := pluginGit.GetDynamicGitUrl(taskCtx, op.ConnectionId, op.Url)
84+
if err != nil {
85+
return nil, errors.Default.Wrap(err, "failed to get Git URL")
86+
}
87+
88+
op.Url = gitUrl
89+
} else {
90+
log.Printf("Plugin does not implement DynamicGitUrl interface for plugin: %s", op.PluginName)
91+
}
92+
}
93+
7194
parsedURL, err := giturls.Parse(op.Url)
7295
if err != nil {
7396
return nil, errors.BadInput.Wrap(err, "failed to parse git url")
@@ -122,3 +145,7 @@ func (p GitExtractor) Close(taskCtx plugin.TaskContext) errors.Error {
122145
func (p GitExtractor) RootPkgPath() string {
123146
return "github.com/apache/incubator-devlake/plugins/gitextractor"
124147
}
148+
149+
func (p GitExtractor) TestConnection(id uint64) errors.Error {
150+
return nil
151+
}

‎backend/plugins/gitextractor/parser/taskdata.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,4 +45,6 @@ type GitExtractorOptions struct {
4545
SkipCommitStat *bool `json:"skipCommitStat" mapstructure:"skipCommitStat" comment:"skip all commit stat including added/deleted lines and commit files as well"`
4646
SkipCommitFiles *bool `json:"skipCommitFiles" mapstructure:"skipCommitFiles"`
4747
NoShallowClone bool `json:"noShallowClone" mapstructure:"noShallowClone"`
48+
ConnectionId uint64 `json:"connectionId" mapstructure:"connectionId,omitempty"`
49+
PluginName string `json:"pluginName" mapstructure:"pluginName,omitempty"`
4850
}

‎backend/plugins/github/api/blueprint_v200.go‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -128,11 +128,13 @@ func makeDataSourcePipelinePlanV200(
128128
stage = append(stage, &coreModels.PipelineTask{
129129
Plugin: "gitextractor",
130130
Options: map[string]interface{}{
131-
"url": cloneUrl.String(),
132-
"name": githubRepo.FullName,
133-
"fullName": githubRepo.FullName,
134-
"repoId": didgen.NewDomainIdGenerator(&models.GithubRepo{}).Generate(connection.ID, githubRepo.GithubId),
135-
"proxy": connection.Proxy,
131+
"url": cloneUrl.String(),
132+
"name": githubRepo.FullName,
133+
"fullName": githubRepo.FullName,
134+
"repoId": didgen.NewDomainIdGenerator(&models.GithubRepo{}).Generate(connection.ID, githubRepo.GithubId),
135+
"proxy": connection.Proxy,
136+
"connectionId": githubRepo.ConnectionId,
137+
"pluginName": "github",
136138
},
137139
})
138140

‎backend/plugins/github/impl/impl.go‎

Lines changed: 55 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ package impl
1919

2020
import (
2121
"fmt"
22+
"strings"
2223

2324
"github.com/apache/incubator-devlake/helpers/pluginhelper/subtaskmeta/sorter"
2425

@@ -243,9 +244,41 @@ func (p Github) Close(taskCtx plugin.TaskContext) errors.Error {
243244
return nil
244245
}
245246

247+
func (p Github) GetDynamicGitUrl(taskCtx plugin.TaskContext, connectionId uint64, repoUrl string) (string, errors.Error) {
248+
connectionHelper := helper.NewConnectionHelper(
249+
taskCtx,
250+
nil,
251+
p.Name(),
252+
)
253+
254+
connection := &models.GithubConnection{}
255+
err := connectionHelper.FirstById(connection, connectionId)
256+
if err != nil {
257+
return "", errors.Default.Wrap(err, "unable to get github connection by the given connection ID")
258+
}
259+
260+
apiClient, err := helper.NewApiClient(taskCtx.GetContext(), connection.GetEndpoint(), nil, 0, connection.GetProxy(), taskCtx)
261+
if err != nil {
262+
return "", err
263+
}
264+
265+
err = connection.PrepareApiClient(apiClient)
266+
if err != nil {
267+
return "", err
268+
}
269+
270+
newUrl, err := replaceAcessTokenInUrl(repoUrl, connection.Token)
271+
if err != nil {
272+
return "", err
273+
}
274+
275+
return newUrl, nil
276+
}
277+
246278
func EnrichOptions(taskCtx plugin.TaskContext,
247279
op *tasks.GithubOptions,
248-
apiClient *helper.ApiClient) errors.Error {
280+
apiClient *helper.ApiClient,
281+
) errors.Error {
249282
var githubRepo models.GithubRepo
250283
// validate the op and set name=owner/repo if this is from advanced mode or bpV100
251284
err := tasks.ValidateTaskOptions(op)
@@ -310,3 +343,24 @@ func convertApiRepoToScope(repo *tasks.GithubApiRepo, connectionId uint64) *mode
310343
scope.CloneUrl = repo.CloneUrl
311344
return &scope
312345
}
346+
347+
func replaceAcessTokenInUrl(gitURL, newCredential string) (string, errors.Error) {
348+
atIndex := strings.Index(gitURL, "@")
349+
if atIndex == -1 {
350+
return "", errors.Default.New("Invalid Git URL")
351+
}
352+
353+
protocolIndex := strings.Index(gitURL, "://")
354+
if protocolIndex == -1 {
355+
return "", errors.Default.New("Invalid Git URL")
356+
}
357+
358+
// Extract the base URL (e.g., "https://git:")
359+
baseURL := gitURL[:protocolIndex+7]
360+
361+
repoURL := gitURL[atIndex+1:]
362+
363+
modifiedURL := fmt.Sprintf("%s%s@%s", baseURL, newCredential, repoURL)
364+
365+
return modifiedURL, nil
366+
}

‎backend/plugins/gitlab/api/blueprint_V200_test.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,9 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
130130
Name: gitlabProjectName,
131131
PathWithNamespace: pathWithNamespace,
132132
HttpUrlToRepo: httpUrlToRepo,
133+
Scope: common.Scope{
134+
ConnectionId: connectionID,
135+
},
133136
},
134137
ScopeConfig: scopeConfig,
135138
},
@@ -166,6 +169,8 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
166169
"name": gitlabProjectName,
167170
"fullName": pathWithNamespace,
168171
"url": "https://git:nddtf@this_is_cloneUrl",
172+
"connectionId": connectionID,
173+
"pluginName": pluginName,
169174
},
170175
},
171176
},

‎backend/plugins/gitlab/api/blueprint_v200.go‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -135,11 +135,13 @@ func makePipelinePlanV200(
135135
stage = append(stage, &coreModels.PipelineTask{
136136
Plugin: "gitextractor",
137137
Options: map[string]interface{}{
138-
"url": cloneUrl.String(),
139-
"name": gitlabProject.Name,
140-
"fullName": gitlabProject.PathWithNamespace,
141-
"repoId": didgen.NewDomainIdGenerator(&models.GitlabProject{}).Generate(connection.ID, gitlabProject.GitlabId),
142-
"proxy": connection.Proxy,
138+
"url": cloneUrl.String(),
139+
"name": gitlabProject.Name,
140+
"fullName": gitlabProject.PathWithNamespace,
141+
"repoId": didgen.NewDomainIdGenerator(&models.GitlabProject{}).Generate(connection.ID, gitlabProject.GitlabId),
142+
"proxy": connection.Proxy,
143+
"connectionId": gitlabProject.ConnectionId,
144+
"pluginName": "gitlab",
143145
},
144146
})
145147
}

0 commit comments

Comments
 (0)