feat: add support for Release Pipelines (Classic Pipelin…

backend/plugins/azuredevops_go/impl/impl.go

@@ -88,6 +88,8 @@ func (p Azuredevops) GetTablesInfo() []dal.Tabler {
 		&models.AzuredevopsPrLabel{},
 		&models.AzuredevopsProject{},
 		&models.AzuredevopsPullRequest{},
+		&models.AzuredevopsRelease{},
+		&models.AzuredevopsReleaseDeployment{},
 		&models.AzuredevopsRepo{},
 		&models.AzuredevopsRepoCommit{},
 		&models.AzuredevopsScopeConfig{},

backend/plugins/azuredevops_go/models/migrationscripts/20241231_add_release_tables.go

@@ -0,0 +1,45 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+	"github.com/apache/incubator-devlake/core/context"
+	"github.com/apache/incubator-devlake/core/errors"
+	"github.com/apache/incubator-devlake/helpers/migrationhelper"
+	"github.com/apache/incubator-devlake/plugins/azuredevops_go/models/migrationscripts/archived"
+)
+
+type addReleaseTables struct {
+}
+
+func (u *addReleaseTables) Up(basicRes context.BasicRes) errors.Error {
+	err := migrationhelper.AutoMigrateTables(
+		basicRes,
+		&archived.AzuredevopsRelease{},
+		&archived.AzuredevopsReleaseDeployment{},
+	)
+	return err
+}
+
+func (*addReleaseTables) Version() uint64 {
+	return 20241231000001
+}
+
+func (*addReleaseTables) Name() string {
+	return "Add Azure DevOps Release Pipeline tables"
+}

backend/plugins/azuredevops_go/models/migrationscripts/archived/release.go

@@ -0,0 +1,69 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package archived
+
+import (
+	"github.com/apache/incubator-devlake/core/models/migrationscripts/archived"
+	"time"
+)
+
+type AzuredevopsRelease struct {
+	archived.NoPKModel
+
+	ConnectionId          uint64 `gorm:"primaryKey"`
+	AzuredevopsId         int    `gorm:"primaryKey"`
+	ProjectId             string `gorm:"type:varchar(255)"`
+	Name                  string `gorm:"type:varchar(255)"`
+	Status                string `gorm:"type:varchar(100)"`
+	ReleaseDefinitionId   int
+	ReleaseDefinitionName string `gorm:"type:varchar(255)"`
+	Description           string `gorm:"type:text"`
+	CreatedOn             *time.Time
+	ModifiedOn            *time.Time
+}
+
+func (AzuredevopsRelease) TableName() string {
+	return "_tool_azuredevops_go_releases"
+}
+
+type AzuredevopsReleaseDeployment struct {
+	archived.NoPKModel
+
+	ConnectionId     uint64 `gorm:"primaryKey"`
+	AzuredevopsId    int    `gorm:"primaryKey"`
+	ReleaseId        int    `gorm:"index"`
+	ProjectId        string `gorm:"type:varchar(255)"`
+	Name             string `gorm:"type:varchar(255)"`
+	Status           string `gorm:"type:varchar(100)"`
+	OperationStatus  string `gorm:"type:varchar(100)"`
+	DeploymentStatus string `gorm:"type:varchar(100)"`
+	DefinitionName   string `gorm:"type:varchar(255)"`
+	DefinitionId     int
+	EnvironmentId    int
+	EnvironmentName  string `gorm:"type:varchar(255)"`
+	AttemptNumber    int
+	Reason           string `gorm:"type:varchar(100)"`
+	QueuedOn         *time.Time
+	StartedOn        *time.Time
+	CompletedOn      *time.Time
+	LastModifiedOn   *time.Time
+}
+
+func (AzuredevopsReleaseDeployment) TableName() string {
+	return "_tool_azuredevops_go_release_deployments"
+}

backend/plugins/azuredevops_go/models/migrationscripts/register.go

@@ -26,5 +26,6 @@ func All() []plugin.MigrationScript {
 	return []plugin.MigrationScript{
 		new(addInitTables),
 		new(extendRepoTable),
+		new(addReleaseTables),
 	}
 }

backend/plugins/azuredevops_go/models/release.go

@@ -0,0 +1,151 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import (
+	"github.com/apache/incubator-devlake/core/models/common"
+	"time"
+)
+
+// AzuredevopsRelease represents a release from Azure DevOps Release Pipelines (Classic)
+type AzuredevopsRelease struct {
+	common.NoPKModel
+
+	ConnectionId          uint64 `gorm:"primaryKey"`
+	AzuredevopsId         int    `json:"id" gorm:"primaryKey"`
+	ProjectId             string
+	Name                  string
+	Status                string
+	ReleaseDefinitionId   int
+	ReleaseDefinitionName string
+	Description           string
+	CreatedOn             *time.Time
+	ModifiedOn            *time.Time
+}
+
+func (AzuredevopsRelease) TableName() string {
+	return "_tool_azuredevops_go_releases"
+}
+
+// AzuredevopsReleaseDeployment represents a deployment (environment) within a release
+type AzuredevopsReleaseDeployment struct {
+	common.NoPKModel
+
+	ConnectionId      uint64 `gorm:"primaryKey"`
+	AzuredevopsId     int    `json:"id" gorm:"primaryKey"`
+	ReleaseId         int    `gorm:"index"`
+	ProjectId         string
+	Name              string
+	Status            string
+	OperationStatus   string
+	DeploymentStatus  string
+	DefinitionName    string
+	DefinitionId      int
+	EnvironmentId     int
+	EnvironmentName   string
+	AttemptNumber     int
+	Reason            string
+	QueuedOn          *time.Time
+	StartedOn         *time.Time
+	CompletedOn       *time.Time
+	LastModifiedOn    *time.Time
+}
+
+func (AzuredevopsReleaseDeployment) TableName() string {
+	return "_tool_azuredevops_go_release_deployments"
+}
+
+// AzuredevopsApiRelease is the API response structure from Azure DevOps Release API
+type AzuredevopsApiRelease struct {
+	Id                int        `json:"id"`
+	Name              string     `json:"name"`
+	Status            string     `json:"status"`
+	CreatedOn         *time.Time `json:"createdOn"`
+	ModifiedOn        *time.Time `json:"modifiedOn"`
+	Description       string     `json:"description"`
+	ReleaseDefinition struct {
+		Id   int    `json:"id"`
+		Name string `json:"name"`
+		Url  string `json:"url"`
+		Path string `json:"path"`
+	} `json:"releaseDefinition"`
+	Environments []AzuredevopsApiReleaseEnvironment `json:"environments"`
+	ProjectReference struct {
+		Id   string `json:"id"`
+		Name string `json:"name"`
+	} `json:"projectReference"`
+}
+
+// AzuredevopsApiReleaseEnvironment represents an environment in the release
+type AzuredevopsApiReleaseEnvironment struct {
+	Id              int        `json:"id"`
+	ReleaseId       int        `json:"releaseId"`
+	Name            string     `json:"name"`
+	Status          string     `json:"status"`
+	DeploySteps     []AzuredevopsApiDeployStep `json:"deploySteps"`
+	PreDeployApprovals []struct {
+		Status string `json:"status"`
+	} `json:"preDeployApprovals"`
+	PostDeployApprovals []struct {
+		Status string `json:"status"`
+	} `json:"postDeployApprovals"`
+}
+
+// AzuredevopsApiDeployStep represents a deployment step
+type AzuredevopsApiDeployStep struct {
+	Id               int        `json:"id"`
+	DeploymentId     int        `json:"deploymentId"`
+	Attempt          int        `json:"attempt"`
+	Reason           string     `json:"reason"`
+	Status           string     `json:"status"`
+	OperationStatus  string     `json:"operationStatus"`
+	QueuedOn         *time.Time `json:"queuedOn"`
+	LastModifiedOn   *time.Time `json:"lastModifiedOn"`
+	HasStarted       bool       `json:"hasStarted"`
+}
+
+// AzuredevopsApiDeployment is the API response structure for deployments
+type AzuredevopsApiDeployment struct {
+	Id                int        `json:"id"`
+	Release           struct {
+		Id   int    `json:"id"`
+		Name string `json:"name"`
+	} `json:"release"`
+	ReleaseDefinition struct {
+		Id   int    `json:"id"`
+		Name string `json:"name"`
+		Path string `json:"path"`
+	} `json:"releaseDefinition"`
+	ReleaseEnvironment struct {
+		Id   int    `json:"id"`
+		Name string `json:"name"`
+	} `json:"releaseEnvironment"`
+	ProjectReference struct {
+		Id   string `json:"id"`
+		Name string `json:"name"`
+	} `json:"projectReference"`
+	DefinitionEnvironmentId int        `json:"definitionEnvironmentId"`
+	Attempt                 int        `json:"attempt"`
+	Reason                  string     `json:"reason"`
+	DeploymentStatus        string     `json:"deploymentStatus"`
+	OperationStatus         string     `json:"operationStatus"`
+	QueuedOn                *time.Time `json:"queuedOn"`
+	StartedOn               *time.Time `json:"startedOn"`
+	CompletedOn             *time.Time `json:"completedOn"`
+	LastModifiedOn          *time.Time `json:"lastModifiedOn"`
+}

backend/plugins/azuredevops_go/tasks/release_collector.go

@@ -0,0 +1,93 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+	"encoding/json"
+	"net/url"
+	"strconv"
+	"time"
+
+	"github.com/apache/incubator-devlake/core/errors"
+	"github.com/apache/incubator-devlake/core/plugin"
+	"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+)
+
+func init() {
+	RegisterSubtaskMeta(&CollectReleasesMeta)
+}
+
+const RawReleaseTable = "azuredevops_go_api_releases"
+
+var CollectReleasesMeta = plugin.SubTaskMeta{
+	Name:             "collectApiReleases",
+	EntryPoint:       CollectReleases,
+	EnabledByDefault: true,
+	Description:      "Collect Release Pipeline data from Azure DevOps Release API (Classic Pipelines)",
+	DomainTypes:      []string{plugin.DOMAIN_TYPE_CICD},
+	ProductTables:    []string{RawReleaseTable},
+}
+
+func CollectReleases(taskCtx plugin.SubTaskContext) errors.Error {
+	rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RawReleaseTable)
+
+	collector, err := api.NewStatefulApiCollectorForFinalizableEntity(api.FinalizableApiCollectorArgs{
+		RawDataSubTaskArgs: *rawDataSubTaskArgs,
+		ApiClient:          data.ApiClient,
+		CollectNewRecordsByList: api.FinalizableApiCollectorListArgs{
+			GetNextPageCustomData: ExtractContToken,
+			PageSize:              100,
+			FinalizableApiCollectorCommonArgs: api.FinalizableApiCollectorCommonArgs{
+				// Azure DevOps Release API uses a different base URL: vsrm.dev.azure.com
+				UrlTemplate: "https://vsrm.dev.azure.com/{{ .Params.OrganizationId }}/{{ .Params.ProjectId }}/_apis/release/releases?api-version=7.1",
+				Query: func(reqData *api.RequestData, createdAfter *time.Time) (url.Values, errors.Error) {
+					query := url.Values{}
+					query.Set("$top", strconv.Itoa(reqData.Pager.Size))
+					query.Set("$expand", "environments")
+					if reqData.CustomData != nil {
+						pag := reqData.CustomData.(CustomPageDate)
+						query.Set("continuationToken", pag.ContinuationToken)
+					}
+
+					if createdAfter != nil {
+						query.Set("minCreatedTime", createdAfter.Format(time.RFC3339))
+					}
+					return query, nil
+				},
+				ResponseParser: ParseRawMessageFromValue,
+				AfterResponse:  change203To401,
+			},
+			GetCreated: func(item json.RawMessage) (time.Time, errors.Error) {
+				var release struct {
+					CreatedOn time.Time `json:"createdOn"`
+				}
+				err := json.Unmarshal(item, &release)
+				if err != nil {
+					return time.Time{}, errors.BadInput.Wrap(err, "failed to unmarshal Azure DevOps Release")
+				}
+				return release.CreatedOn, nil
+			},
+		},
+	})
+
+	if err != nil {
+		return err
+	}
+
+	return collector.Execute()
+}