fix: dora refdiff and change lead time logic (#4907)

Author: abeizn

backend/plugins/dora/tasks/change_lead_time_calculator.go

@@ -18,15 +18,16 @@ limitations under the License.
 package tasks
 
 import (
+	"reflect"
+	"time"
+
 	"github.com/apache/incubator-devlake/core/dal"
 	"github.com/apache/incubator-devlake/core/errors"
 	"github.com/apache/incubator-devlake/core/models/domainlayer/code"
 	"github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain"
 	"github.com/apache/incubator-devlake/core/models/domainlayer/devops"
 	"github.com/apache/incubator-devlake/core/plugin"
 	"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
-	"reflect"
-	"time"
 )
 
 // CalculateChangeLeadTime calculates change lead time for a project.

backend/plugins/refdiff/e2e/raw_tables/cicd_pipeline_commits.csv

@@ -4,3 +4,5 @@ github:GithubRun:1:2,commit_sha2,main,github:GithubRepo:1:484251804,""
 github:GithubRun:1:3,commit_sha3,fix-edit-url,github:GithubRepo:1:484251804,""
 github:GithubRun:1:4,commit_sha4,main,github:GithubRepo:1:484251804,""
 github:GithubRun:1:5,commit_sha5,main,github:GithubRepo:1:484251804,""
+github:GithubRun:1:6,commit_sha6,main,github:GithubRepo:1:484251804,""
+github:GithubRun:1:7,commit_sha7,main,github:GithubRepo:1:484251804,""

backend/plugins/refdiff/e2e/raw_tables/cicd_pipelines.csv

@@ -3,4 +3,6 @@ github:GithubRun:1:1,Production workflow,SUCCESS,DONE,github:GithubRepo:1:484251
 github:GithubRun:1:2,Production workflow,SUCCESS,DONE,github:GithubRepo:1:484251804
 github:GithubRun:1:3,Build workflow,SUCCESS,DONE,github:GithubRepo:1:484251804
 github:GithubRun:1:4,Production workflow,SUCCESS,DONE,github:GithubRepo:1:484251804
-github:GithubRun:1:5,Production workflow,SUCCESS,DONE,github:GithubRepo:1:484251804
+github:GithubRun:1:5,Production workflow,FAILURE,DONE,github:GithubRepo:1:484251804
+github:GithubRun:1:6,Production workflow,SUCCESS,DONE,github:GithubRepo:1:484251804
+github:GithubRun:1:7,Production workflow,SUCCESS,DONE,github:GithubRepo:1:484251804

backend/plugins/refdiff/e2e/raw_tables/cicd_tasks.csv

@@ -3,4 +3,6 @@ github:GithubJob:1:1,build-and-deploy,github:GithubRun:1:1,SUCCESS,DONE,DEPLOYME
 github:GithubJob:1:2,build-and-deploy,github:GithubRun:1:2,SUCCESS,DONE,DEPLOYMENT,PRODUCTION,""
 github:GithubJob:1:3,build,github:GithubRun:1:3,SUCCESS,DONE,"","",""
 github:GithubJob:1:4,build-and-deploy,github:GithubRun:1:4,SUCCESS,DONE,DEPLOYMENT,PRODUCTION,""
-
+github:GithubJob:1:5,build-and-deploy,github:GithubRun:1:5,FAILURE,DONE,DEPLOYMENT,PRODUCTION,""
+github:GithubJob:1:6,build-and-deploy,github:GithubRun:1:6,SUCCESS,DONE,DEPLOYMENT,STAGING,""
+github:GithubJob:1:7,build-and-deploy,github:GithubRun:1:7,SUCCESS,DONE,"",PRODUCTION,""

backend/plugins/refdiff/e2e/raw_tables/commit_parents.csv

@@ -4,3 +4,5 @@ commit_sha2,commit_sha6
 commit_sha3,commit_sha6
 commit_sha4,commit_sha7
 commit_sha5,commit_sha7
+commit_sha6,commit_sha7
+commit_sha7,commit_sha8

backend/plugins/refdiff/e2e/raw_tables/project_mapping.csv

@@ -1,5 +1,6 @@
 project_name,table,row_id
 project1,repos,github:GithubRepo:1:484251804
+project1,cicd_scopes,github:GithubRepo:1:484251804
 project2,repos,github:GithubRepo:1:384111310
 project3,boards,jira:JiraBoard:1:68
 

backend/plugins/refdiff/e2e/snapshot_tables/commits_diffs.csv

@@ -1,4 +1,3 @@
-commit_sha,new_commit_sha,old_commit_sha,sorting_index
-commit_sha2,commit_sha2,commit_sha1,1
-commit_sha4,commit_sha4,commit_sha2,1
-commit_sha7,commit_sha4,commit_sha2,2
+new_commit_sha,old_commit_sha,commit_sha,sorting_index
+commit_sha2,commit_sha1,commit_sha2,1
+commit_sha4,commit_sha2,commit_sha4,1

backend/plugins/refdiff/tasks/project_deployment_commit_diff_calculator.go

@@ -18,12 +18,13 @@ limitations under the License.
 package tasks
 
 import (
+	"reflect"
+
 	"github.com/apache/incubator-devlake/core/dal"
 	"github.com/apache/incubator-devlake/core/errors"
 	"github.com/apache/incubator-devlake/core/models/domainlayer/code"
 	"github.com/apache/incubator-devlake/core/plugin"
 	"github.com/apache/incubator-devlake/plugins/refdiff/utils"
-	"reflect"
 )
 
 func CommitDiffConvertor(pipelineCommitShaList []string, existFinishedCommitDiff []code.FinishedCommitsDiff) (commitPairs []code.CommitsDiff, finishedCommitDiffs []code.FinishedCommitsDiff) {
@@ -53,8 +54,8 @@ func CalculateProjectDeploymentCommitsDiff(taskCtx plugin.SubTaskContext) errors
 
 	cursorScope, err := db.Cursor(
 		dal.Select("row_id"),
-		dal.From("project_mapping"),
-		dal.Where("project_name = ?", projectName),
+		dal.From("project_mapping pm"),
+		dal.Where("pm.project_name = ? and pm.table = ?", projectName, "cicd_scopes"),
 	)
 	if err != nil {
 		return err
@@ -83,7 +84,7 @@ func CalculateProjectDeploymentCommitsDiff(taskCtx plugin.SubTaskContext) errors
 			dal.From("cicd_tasks ct"),
 			dal.Join("left join cicd_pipelines cp on cp.id = ct.pipeline_id"),
 			dal.Join("left join cicd_pipeline_commits cpc on cpc.pipeline_id = cp.id"),
-			dal.Where("ct.type = ? and commit_sha != ? and repo_id=? ", "DEPLOYMENT", "", scopeId),
+			dal.Where("ct.environment = ? and ct.type = ? and ct.result = ? and commit_sha != ? and repo_id=? ", "PRODUCTION", "DEPLOYMENT", "SUCCESS", "", scopeId),
 			dal.Orderby("ct.started_date"),
 		)
 		if err != nil {