fix: Fix azure devops migration from 0.17 to 0.18 (#5999)

Cherry-pick of #5957

* fix: Fix AzDO connection migration 0.17 -> 0.18

In 0.17, all connection fields are encrypted.
In 0.18 only `token` is, so we need a migration of AzDo connection to decrypt all fields but `token`.

* fix: Allow setting entities=None to ScopeConfig

Allow setting `entities` field to `None` in `ScopeConfig` and fallback to default value.

* feat: Add RenameColumn migration op

* fix: Add missing AzDO migrations from 0.17 to 0.18

Now that migrations have been made manual, we need to add all the migrations for
all that changed from 0.17 to 0.18 that were automatic before...

Those are all required migrations collected with `git diff v0.17.0 v0.18.0-beta7 -- backend/python/plugins/azuredevops/azuredevops/models.py`:
* Rename GitRepository.transformation_rule_id -> scope_config_id
* Add GitRepository.provider
* Rename Job.startTime -> start_time
* Rename Job.finishTime -> start_time

---------

Co-authored-by: Camille Teruel <camille.teruel@meri.co>

Author: CamilleTeruel

backend/python/plugins/azuredevops/azuredevops/models.py

@@ -158,7 +158,11 @@ def add_raw_data_params_table_to_scope(b: MigrationScriptBuilder):
 
 @migration(20230802000001, name="rename startTime/finishTime to start_time/finish_time")
 def rename_starttime_and_finishtime_for_job(b: MigrationScriptBuilder):
-    b.execute(f'ALTER TABLE _tool_azuredevops_jobs RENAME COLUMN startTime TO start_time', Dialect.MYSQL, ignore_error=True)
-    b.execute(f'ALTER TABLE _tool_azuredevops_jobs RENAME COLUMN finishTime TO finish_time', Dialect.MYSQL, ignore_error=True)
-    b.execute(f'ALTER TABLE _tool_azuredevops_jobs RENAME COLUMN `startTime` TO start_time', Dialect.POSTGRESQL, ignore_error=True)
-    b.execute(f'ALTER TABLE _tool_azuredevops_jobs RENAME COLUMN `finishTime` TO finish_time', Dialect.POSTGRESQL, ignore_error=True)
+    b.rename_column('_tool_azuredevops_jobs', 'startTime', 'start_time')
+    b.rename_column('_tool_azuredevops_jobs', 'finishTime', 'finish_time')
+
+
+@migration(20230825150421, name="add missing migrations from 0.17 to 0.18")
+def add_missing_migrations_0_17_to_0_18(b: MigrationScriptBuilder):
+    b.rename_column('_tool_azuredevops_gitrepositories', 'transformation_rule_id', 'scope_config_id')
+    b.add_column('_tool_azuredevops_gitrepositories', 'provider', 'varchar(255)')

backend/python/pydevlake/pydevlake/model.py

@@ -87,6 +87,12 @@ class ScopeConfig(ToolTable, Model):
     name: str = Field(default="default")
     domain_types: list[DomainType] = Field(default=list(DomainType), alias="entities")
 
+    @validator('domain_types', pre=True, always=True)
+    def set_default_domain_types(cls, v):
+        if v is None:
+            return list(DomainType)
+        return v
+
 
 class RawModel(SQLModel):
     id: int = Field(primary_key=True)

backend/server/services/remote/models/migration.go

@@ -97,6 +97,27 @@ func (o DropTableOperation) Execute(dal dal.Dal) errors.Error {
 
 var _ Operation = (*DropTableOperation)(nil)
 
+type RenameColumnOperation struct {
+	Table   string `json:"table"`
+	OldName string `json:"old_name"`
+	NewName string `json:"new_name"`
+}
+
+func (o RenameColumnOperation) Execute(dal dal.Dal) errors.Error {
+	if !dal.HasColumn(o.Table, o.OldName) {
+		return nil
+	}
+	if dal.HasColumn(o.Table, o.NewName) {
+		err := dal.DropColumns(o.Table, o.NewName)
+		if err != nil {
+			return err
+		}
+	}
+	return dal.RenameColumn(o.Table, o.OldName, o.NewName)
+}
+
+var _ Operation = (*RenameTableOperation)(nil)
+
 type RenameTableOperation struct {
 	OldName string `json:"old_name"`
 	NewName string `json:"new_name"`
@@ -155,10 +176,12 @@ func (s *RemoteMigrationScript) UnmarshalJSON(data []byte) error {
 			operation = &DropColumnOperation{}
 		case "drop_table":
 			operation = &DropTableOperation{}
+		case "rename_column":
+			operation = &RenameColumnOperation{}
 		case "rename_table":
 			operation = &RenameTableOperation{}
 		default:
-			return errors.BadInput.New("unsupported operation type")
+			return errors.BadInput.New("unsupported operation type: " + operationType)
 		}
 		err = json.Unmarshal(operationRaw, operation)
 		if err != nil {

backend/server/services/remote/models/migrationscripts/azuredevops/azuredevops_20230714_decrypt_azdo_fields.go

@@ -0,0 +1,96 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package azuredevops
+
+import (
+	"github.com/apache/incubator-devlake/core/context"
+	"github.com/apache/incubator-devlake/core/errors"
+	"github.com/apache/incubator-devlake/core/models/migrationscripts/archived"
+	"github.com/apache/incubator-devlake/core/plugin"
+	"github.com/apache/incubator-devlake/helpers/migrationhelper"
+)
+
+var _ plugin.MigrationScript = (*DecryptConnectionFields)(nil)
+
+type azureDevopsConnection20230825 struct {
+	archived.Model
+	Name         string
+	Token        string
+	Proxy        *string
+	Organization *string
+}
+
+type DecryptConnectionFields struct{}
+
+func (script *DecryptConnectionFields) Up(basicRes context.BasicRes) errors.Error {
+	encryptionSecret := basicRes.GetConfig(plugin.EncodeKeyEnvStr)
+	if encryptionSecret == "" {
+		return errors.BadInput.New("invalid encryptionSecret")
+	}
+
+	err := migrationhelper.TransformColumns(
+		basicRes,
+		script,
+		"_tool_azuredevops_azuredevopsconnections",
+		[]string{"name", "proxy", "organization"},
+		func(src *azureDevopsConnection20230825) (*azureDevopsConnection20230825, errors.Error) {
+			encName := src.Name
+			name, err := plugin.Decrypt(encryptionSecret, encName)
+			if err != nil {
+				return src, nil
+			}
+			src.Name = name
+
+			if src.Proxy != nil {
+				encProxy := *src.Proxy
+				decProxy, err := plugin.Decrypt(encryptionSecret, encProxy)
+				if err != nil {
+					return src, nil
+				}
+				if decProxy == "" {
+					src.Proxy = nil
+				} else {
+					src.Proxy = &decProxy
+				}
+			}
+
+			if src.Organization != nil {
+				encOrg := *src.Organization
+				decOrg, err := plugin.Decrypt(encryptionSecret, encOrg)
+				if err != nil {
+					return src, nil
+				}
+				if decOrg == "" {
+					src.Organization = nil
+				} else {
+					src.Organization = &decOrg
+				}
+			}
+			return src, nil
+		},
+	)
+	return err
+}
+
+func (*DecryptConnectionFields) Version() uint64 {
+	return 20230825090504
+}
+
+func (script *DecryptConnectionFields) Name() string {
+	return "Decrypt Azure DevOps connection fields"
+}

backend/server/services/remote/models/migrationscripts/register.go

@@ -25,6 +25,7 @@ import (
 var allMigrations = map[string][]plugin.MigrationScript{
 	"azuredevops": {
 		&azuredevops.AddRawDataForScope{},
+		&azuredevops.DecryptConnectionFields{},
 	},
 }