Merge pull request #1470 from merico-dev/v0.9-gitlab-merge-request-commits-convert

feat: convert merge request commits

Author: klesh

plugins/gitlab/gitlab.go

@@ -3,11 +3,12 @@ package main // must be main for plugin entry point
 import (
 	"context"
 	"fmt"
-	"github.com/merico-dev/lake/config"
-	errors "github.com/merico-dev/lake/errors"
 	"os"
 	"strconv"
 
+	"github.com/merico-dev/lake/config"
+	errors "github.com/merico-dev/lake/errors"
+
 	"github.com/merico-dev/lake/logger" // A pseudo type for Plugin Interface implementation
 	lakeModels "github.com/merico-dev/lake/models"
 	"github.com/merico-dev/lake/plugins/core"
@@ -88,17 +89,18 @@ func (plugin Gitlab) Execute(options map[string]interface{}, progress chan<- flo
 	}
 	if len(tasksToRun) == 0 {
 		tasksToRun = map[string]bool{
-			"collectPipelines": true,
-			"collectCommits":   true,
-			"CollectTags":      true,
-			"collectMrs":       true,
-			"collectMrNotes":   true,
-			"collectMrCommits": true,
-			"enrichMrs":        true,
-			"convertProjects":  true,
-			"convertMrs":       true,
-			"convertCommits":   true,
-			"convertNotes":     true,
+			"collectPipelines":  true,
+			"collectCommits":    true,
+			"CollectTags":       true,
+			"collectMrs":        true,
+			"collectMrNotes":    true,
+			"collectMrCommits":  true,
+			"enrichMrs":         true,
+			"convertProjects":   true,
+			"convertMrs":        true,
+			"convertCommits":    true,
+			"convertMrsCommits": true,
+			"convertNotes":      true,
 		}
 	}
 
@@ -215,6 +217,16 @@ func (plugin Gitlab) Execute(options map[string]interface{}, progress chan<- flo
 			}
 		}
 	}
+	if tasksToRun["convertMrsCommits"] {
+		progress <- 0.85
+		err = tasks.ConvertMergeRequestCommits(projectIdInt)
+		if err != nil {
+			return &errors.SubTaskError{
+				SubTaskName: "convertMrsCommits",
+				Message:     err.Error(),
+			}
+		}
+	}
 	if tasksToRun["convertNotes"] {
 		progress <- 0.9
 		err = tasks.ConvertNotes(ctx, projectIdInt)

plugins/gitlab/tasks/gitlab_merge_request_commits_converter.go

@@ -0,0 +1,54 @@
+package tasks
+
+import (
+	lakeModels "github.com/merico-dev/lake/models"
+	"github.com/merico-dev/lake/models/domainlayer/code"
+	"github.com/merico-dev/lake/models/domainlayer/didgen"
+	gitlabModels "github.com/merico-dev/lake/plugins/gitlab/models"
+	"gorm.io/gorm/clause"
+)
+
+func ConvertMergeRequestCommits(projectId int) error {
+	gitlabMergeRequestCommit := &gitlabModels.GitlabMergeRequestCommit{}
+	cursor, err := lakeModels.Db.Model(&gitlabMergeRequestCommit).
+		Joins(`left join gitlab_merge_requests on gitlab_merge_requests.gitlab_id = gitlab_merge_request_commits.merge_request_id`).
+		Where("gitlab_merge_requests.project_id = ?", projectId).
+		Order("merge_request_id ASC").Rows()
+	if err != nil {
+		return err
+	}
+	defer cursor.Close()
+	var pullRequestId int
+	domainPullRequestId := ""
+	domainIdGenerator := didgen.NewDomainIdGenerator(&gitlabModels.GitlabMergeRequest{})
+
+	for cursor.Next() {
+		err = lakeModels.Db.ScanRows(cursor, gitlabMergeRequestCommit)
+
+		if pullRequestId != gitlabMergeRequestCommit.MergeRequestId {
+			domainPullRequestId = domainIdGenerator.Generate(gitlabMergeRequestCommit.MergeRequestId)
+			err := lakeModels.Db.Where("pull_request_id = ?",
+				domainPullRequestId).Delete(&code.PullRequestCommit{}).Error
+			if err != nil {
+				return err
+			}
+			pullRequestId = gitlabMergeRequestCommit.MergeRequestId
+		}
+		if err != nil {
+			return err
+		}
+
+		domainPrcommit := &code.PullRequestCommit{
+			CommitSha:     gitlabMergeRequestCommit.CommitSha,
+			PullRequestId: domainPullRequestId,
+		}
+
+		err = lakeModels.Db.Clauses(clause.OnConflict{
+			DoNothing: true,
+		}).Create(domainPrcommit).Error
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}