fix: CircleCI column naming inconsistency (#8497)

backend/plugins/circleci/e2e/project_test.go

@@ -61,9 +61,8 @@ func TestCircleciProject(t *testing.T) {
 	dataflowTester.VerifyTableWithOptions(
 		devops.CicdScope{},
 		e2ehelper.TableOptions{
-			CSVRelPath:   "./snapshot_tables/cicd_scopes.csv",
-			IgnoreFields: []string{"created_date"},
-			IgnoreTypes:  []interface{}{domainlayer.DomainEntity{}},
+			CSVRelPath:  "./snapshot_tables/cicd_scopes.csv",
+			IgnoreTypes: []interface{}{domainlayer.DomainEntity{}},
 		},
 	)
 }

backend/plugins/circleci/e2e/snapshot_tables/_tool_circleci_workflows.csv

@@ -1,4 +1,4 @@
-connection_id,id,project_slug,pipeline_id,canceled_by,name,errored_by,tag,status,started_by,pipeline_number,created_date,stopped_date,duration_sec
+connection_id,id,project_slug,pipeline_id,canceled_by,name,errored_by,tag,status,started_by,pipeline_number,created_at,stopped_at,duration_sec
 1,6731159f-5275-4bfa-ba70-39d343d63814,github/coldgust/coldgust.github.io,23622ee4-e150-4920-9d66-81533fa765a4,,workflow,,,failed,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,5,2023-03-25T17:52:15.000+00:00,2023-03-25T17:52:23.000+00:00,8
 1,7370985a-9de3-4a47-acbc-e6a1fe8e5812,github/coldgust/coldgust.github.io,866e967d-f826-4470-aed6-fc0c92e98703,,workflow,,,failed,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,7,2023-03-25T17:56:24.000+00:00,2023-03-25T17:56:43.000+00:00,19
 1,89054eb2-8e85-4f5c-9a93-66d753a0e970,github/coldgust/coldgust.github.io,625ca634-68fe-4515-91f0-7ba8af51dc99,,say-hello-workflow,,,success,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,3,2023-03-25T17:39:23.000+00:00,2023-03-25T17:39:28.000+00:00,5

backend/plugins/circleci/e2e/snapshot_tables/cicd_pipelines.csv

@@ -1,10 +1,10 @@
-id,name,display_title,url,result,status,original_status,original_result,type,duration_sec,queued_duration_sec,environment,queued_date,started_date,cicd_scope_id
-circleci:CircleciWorkflow:1:6731159f-5275-4bfa-ba70-39d343d63814,workflow,workflow#5,,FAILURE,DONE,failed,,,8,,PRODUCTION,,2023-03-25T17:52:15.000+00:00,circleci:CircleciProject:1:github/coldgust/coldgust.github.io
-circleci:CircleciWorkflow:1:7370985a-9de3-4a47-acbc-e6a1fe8e5812,workflow,workflow#7,,FAILURE,DONE,failed,,,19,,PRODUCTION,,2023-03-25T17:56:24.000+00:00,circleci:CircleciProject:1:github/coldgust/coldgust.github.io
-circleci:CircleciWorkflow:1:89054eb2-8e85-4f5c-9a93-66d753a0e970,say-hello-workflow,say-hello-workflow#3,,SUCCESS,DONE,success,,,5,,PRODUCTION,,2023-03-25T17:39:23.000+00:00,circleci:CircleciProject:1:github/coldgust/coldgust.github.io
-circleci:CircleciWorkflow:1:8971a56b-5547-4824-94dd-07bb467524c5,say-hello-workflow,say-hello-workflow#2,,SUCCESS,DONE,success,,,5,,PRODUCTION,,2023-03-25T17:12:18.000+00:00,circleci:CircleciProject:1:github/coldgust/coldgust.github.io
-circleci:CircleciWorkflow:1:8fe60291-68f7-40e2-acec-d99bff4da713,say-hello-workflow,say-hello-workflow#1,,SUCCESS,DONE,success,,,4,,PRODUCTION,,2023-03-25T17:12:18.000+00:00,circleci:CircleciProject:1:github/coldgust/coldgust.github.io
-circleci:CircleciWorkflow:1:b9ab7bbe-2f30-4c59-b4e2-eb2005bffb14,workflow,workflow#6,,FAILURE,DONE,failed,,,14,,PRODUCTION,,2023-03-25T17:54:09.000+00:00,circleci:CircleciProject:1:github/coldgust/coldgust.github.io
-circleci:CircleciWorkflow:1:c7df82a6-0d2b-4e19-a36a-3f3aa9fd3943,workflow,workflow#4,,FAILURE,DONE,failed,,,5,,PRODUCTION,,2023-03-25T17:50:20.000+00:00,circleci:CircleciProject:1:github/coldgust/coldgust.github.io
-circleci:CircleciWorkflow:1:fc76deef-bcdd-4856-8e96-a8e2d1c5a85f,workflow,workflow#8,,FAILURE,DONE,failed,,,15,,PRODUCTION,,2023-03-25T18:06:13.000+00:00,circleci:CircleciProject:1:github/coldgust/coldgust.github.io
-circleci:CircleciWorkflow:1:fd0bd4f5-264f-4e3c-a151-06153c018f78,workflow,workflow#9,,SUCCESS,DONE,success,,,17,,PRODUCTION,,2023-03-25T18:13:21.000+00:00,circleci:CircleciProject:1:github/coldgust/coldgust.github.io
+id,name,display_title,url,result,status,original_status,original_result,type,duration_sec,queued_duration_sec,environment,created_date,queued_date,started_date,finished_date,cicd_scope_id
+circleci:CircleciWorkflow:1:6731159f-5275-4bfa-ba70-39d343d63814,workflow,workflow#5,,FAILURE,DONE,failed,,,8,,PRODUCTION,2023-03-25T17:52:15.000+00:00,,2023-03-25T17:52:15.000+00:00,2023-03-25T17:52:23.000+00:00,circleci:CircleciProject:1:github/coldgust/coldgust.github.io
+circleci:CircleciWorkflow:1:7370985a-9de3-4a47-acbc-e6a1fe8e5812,workflow,workflow#7,,FAILURE,DONE,failed,,,19,,PRODUCTION,2023-03-25T17:56:24.000+00:00,,2023-03-25T17:56:24.000+00:00,2023-03-25T17:56:43.000+00:00,circleci:CircleciProject:1:github/coldgust/coldgust.github.io
+circleci:CircleciWorkflow:1:89054eb2-8e85-4f5c-9a93-66d753a0e970,say-hello-workflow,say-hello-workflow#3,,SUCCESS,DONE,success,,,5,,PRODUCTION,2023-03-25T17:39:23.000+00:00,,2023-03-25T17:39:23.000+00:00,2023-03-25T17:39:28.000+00:00,circleci:CircleciProject:1:github/coldgust/coldgust.github.io
+circleci:CircleciWorkflow:1:8971a56b-5547-4824-94dd-07bb467524c5,say-hello-workflow,say-hello-workflow#2,,SUCCESS,DONE,success,,,5,,PRODUCTION,2023-03-25T17:12:18.000+00:00,,2023-03-25T17:12:18.000+00:00,2023-03-25T17:12:23.000+00:00,circleci:CircleciProject:1:github/coldgust/coldgust.github.io
+circleci:CircleciWorkflow:1:8fe60291-68f7-40e2-acec-d99bff4da713,say-hello-workflow,say-hello-workflow#1,,SUCCESS,DONE,success,,,4,,PRODUCTION,2023-03-25T17:12:18.000+00:00,,2023-03-25T17:12:18.000+00:00,2023-03-25T17:12:22.000+00:00,circleci:CircleciProject:1:github/coldgust/coldgust.github.io
+circleci:CircleciWorkflow:1:b9ab7bbe-2f30-4c59-b4e2-eb2005bffb14,workflow,workflow#6,,FAILURE,DONE,failed,,,14,,PRODUCTION,2023-03-25T17:54:09.000+00:00,,2023-03-25T17:54:09.000+00:00,2023-03-25T17:54:23.000+00:00,circleci:CircleciProject:1:github/coldgust/coldgust.github.io
+circleci:CircleciWorkflow:1:c7df82a6-0d2b-4e19-a36a-3f3aa9fd3943,workflow,workflow#4,,FAILURE,DONE,failed,,,5,,PRODUCTION,2023-03-25T17:50:20.000+00:00,,2023-03-25T17:50:20.000+00:00,2023-03-25T17:50:25.000+00:00,circleci:CircleciProject:1:github/coldgust/coldgust.github.io
+circleci:CircleciWorkflow:1:fc76deef-bcdd-4856-8e96-a8e2d1c5a85f,workflow,workflow#8,,FAILURE,DONE,failed,,,15,,PRODUCTION,2023-03-25T18:06:13.000+00:00,,2023-03-25T18:06:13.000+00:00,2023-03-25T18:06:28.000+00:00,circleci:CircleciProject:1:github/coldgust/coldgust.github.io
+circleci:CircleciWorkflow:1:fd0bd4f5-264f-4e3c-a151-06153c018f78,workflow,workflow#9,,SUCCESS,DONE,success,,,17,,PRODUCTION,2023-03-25T18:13:21.000+00:00,,2023-03-25T18:13:21.000+00:00,2023-03-25T18:13:38.000+00:00,circleci:CircleciProject:1:github/coldgust/coldgust.github.io

backend/plugins/circleci/e2e/workflow_test.go

@@ -52,9 +52,8 @@ func TestCircleciWorkflow(t *testing.T) {
 	dataflowTester.VerifyTableWithOptions(
 		models.CircleciWorkflow{},
 		e2ehelper.TableOptions{
-			CSVRelPath:   "./snapshot_tables/_tool_circleci_workflows.csv",
-			IgnoreTypes:  []interface{}{common.NoPKModel{}},
-			IgnoreFields: []string{"started_at", "stopped_at"},
+			CSVRelPath:  "./snapshot_tables/_tool_circleci_workflows.csv",
+			IgnoreTypes: []interface{}{common.NoPKModel{}},
 		},
 	)
 
@@ -72,7 +71,7 @@ func TestCircleciWorkflow(t *testing.T) {
 		devops.CICDPipeline{},
 		e2ehelper.TableOptions{
 			CSVRelPath:   "./snapshot_tables/cicd_pipelines.csv",
-			IgnoreFields: []string{"finished_date", "created_date", "is_child"},
+			IgnoreFields: []string{"finished_date", "is_child"},
 			IgnoreTypes:  []interface{}{domainlayer.DomainEntity{}},
 		},
 	)

backend/plugins/circleci/models/migrationscripts/20260208_fix_column_names.go

@@ -0,0 +1,124 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/apache/incubator-devlake/core/context"
+	"github.com/apache/incubator-devlake/core/dal"
+	"github.com/apache/incubator-devlake/core/errors"
+	"gorm.io/gorm/clause"
+)
+
+type fixColumnNames struct{}
+
+type columnRename struct {
+	Old string
+	New string
+}
+
+func (*fixColumnNames) Up(basicRes context.BasicRes) errors.Error {
+	db := basicRes.GetDal()
+
+	// Rename columns for _tool_circleci_pipelines
+	renames := []columnRename{
+		{Old: "created_date", New: "created_at"},
+		{Old: "updated_date", New: "updated_at"},
+	}
+
+	return batchRenameColumns(db, "_tool_circleci_pipelines", renames)
+}
+
+func (*fixColumnNames) Version() uint64 {
+	return 20260208000001
+}
+
+func (*fixColumnNames) Name() string {
+	return "fix circleci column names"
+}
+
+// batchRenameColumns renames multiple columns in a single ALTER TABLE statement
+func batchRenameColumns(db dal.Dal, table string, renames []columnRename) errors.Error {
+	if len(renames) == 0 {
+		return nil
+	}
+
+	// Get all existing column names once to avoid repeated information_schema queries
+	existingColumns := getExistingColumns(db, table)
+
+	// Filter out renames where old column doesn't exist or new column already exists
+	validRenames := filterValidRenamesCached(renames, existingColumns)
+	if len(validRenames) == 0 {
+		return nil
+	}
+
+	var sql string
+	var dialect = db.Dialect()
+
+	if dialect == "postgres" {
+		// PostgreSQL requires separate ALTER TABLE statements for each RENAME COLUMN
+		for _, rename := range validRenames {
+			sql = fmt.Sprintf(`ALTER TABLE "%s" RENAME COLUMN "%s" TO "%s"`, table, rename.Old, rename.New)
+			if err := db.Exec(sql); err != nil {
+				return err
+			}
+		}
+		// Clear PostgreSQL cached plan after all renames
+		_ = db.Exec("SELECT * FROM ? LIMIT 1", clause.Table{Name: table})
+	} else {
+		// MySQL: ALTER TABLE t CHANGE COLUMN a new_name TEXT, CHANGE COLUMN c new_name2 TEXT
+		clauses := make([]string, 0, len(validRenames))
+		for _, rename := range validRenames {
+			clauses = append(clauses, fmt.Sprintf("CHANGE COLUMN `%s` `%s` %s", rename.Old, rename.New, "DATETIME"))
+		}
+		sql = fmt.Sprintf("ALTER TABLE `%s` %s", table, strings.Join(clauses, ", "))
+		if err := db.Exec(sql); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// getExistingColumns fetches all column names for a table in a single query
+func getExistingColumns(db dal.Dal, table string) map[string]bool {
+	columns := make(map[string]bool)
+	columnMetas, err := db.GetColumns(&dal.DefaultTabler{Name: table}, nil)
+	if err != nil {
+		return columns
+	}
+	for _, col := range columnMetas {
+		columns[col.Name()] = true
+	}
+	return columns
+}
+
+// filterValidRenamesCached checks which renames are needed using pre-fetched column map
+func filterValidRenamesCached(renames []columnRename, existingColumns map[string]bool) []columnRename {
+	valid := make([]columnRename, 0, len(renames))
+	for _, rename := range renames {
+		oldExists := existingColumns[rename.Old]
+		newExists := existingColumns[rename.New]
+		if oldExists && !newExists {
+			valid = append(valid, rename)
+		}
+	}
+	return valid
+}

backend/plugins/circleci/models/migrationscripts/archived/pipeline.go

@@ -32,8 +32,8 @@ type CircleciPipeline struct {
 	Trigger           any    `gorm:"type:json"`
 	Vcs               any    `gorm:"type:json"`
 	State             string `gorm:"type:varchar(100)"`
-	UpdatedDate       *time.Time
-	CreatedDate       *time.Time
+	UpdatedAt         *time.Time
+	CreatedAt         *time.Time
 	archived.NoPKModel
 }
 

backend/plugins/circleci/models/migrationscripts/register.go

@@ -24,5 +24,6 @@ func All() []plugin.MigrationScript {
 	return []plugin.MigrationScript{
 		new(addInitTables),
 		new(addFieldsToCircleciJob20231129),
+		new(fixColumnNames),
 	}
 }

backend/plugins/circleci/models/pipeline.go

@@ -49,8 +49,8 @@ type CircleciPipeline struct {
 	ConnectionId      uint64              `gorm:"primaryKey;type:BIGINT" json:"connectionId" mapstructure:"connectionId"`
 	Id                string              `gorm:"primaryKey;type:varchar(100)" json:"id" mapstructure:"id"`
 	ProjectSlug       string              `gorm:"type:varchar(255)" json:"projectSlug" mapstructure:"projectSlug"`
-	UpdatedDate       *common.Iso8601Time `json:"updatedDate" mapstructure:"updatedDate"`
-	CreatedDate       *common.Iso8601Time `json:"createdDate" mapstructure:"createdDate"`
+	UpdatedAt         *common.Iso8601Time `json:"updated_at" mapstructure:"updated_at"`
+	CreatedAt         *common.Iso8601Time `json:"created_at" mapstructure:"created_at"`
 	Number            int64               `json:"number" mapstructure:"number"` // pipeline number within the project?
 	TriggerParameters any                 `gorm:"serializer:json" json:"trigger_parameters" mapstructure:"triggerParameters"`
 	State             string              `gorm:"type:varchar(100)" json:"state" mapstructure:"state"`

backend/plugins/circleci/tasks/job_collector.go

@@ -60,7 +60,7 @@ func CollectJobs(taskCtx plugin.SubTaskContext) errors.Error {
 				}
 
 				if isIncremental {
-					clauses = append(clauses, dal.Where("created_date > ?", createdAfter))
+					clauses = append(clauses, dal.Where("created_at > ?", createdAfter))
 				}
 
 				db := taskCtx.GetDal()

backend/plugins/circleci/tasks/workflow_collector.go

@@ -61,7 +61,7 @@ func CollectWorkflows(taskCtx plugin.SubTaskContext) errors.Error {
 				}
 
 				if isIncremental {
-					clauses = append(clauses, dal.Where("created_date > ?", createdAfter))
+					clauses = append(clauses, dal.Where("created_at > ?", createdAfter))
 				}
 
 				db := taskCtx.GetDal()