Docker preset container registry

Wire.go

@@ -90,6 +90,7 @@ import (
 	jira2 "github.com/devtron-labs/devtron/pkg/jira"
 	"github.com/devtron-labs/devtron/pkg/notifier"
 	"github.com/devtron-labs/devtron/pkg/pipeline"
+	//pipelineCron "github.com/devtron-labs/devtron/pkg/pipeline/cron"
 	history3 "github.com/devtron-labs/devtron/pkg/pipeline/history"
 	repository3 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository"
 	repository5 "github.com/devtron-labs/devtron/pkg/pipeline/repository"
@@ -731,6 +732,9 @@ func InitializeApp() (*App, error) {
 		//	AuthWireSet,
 		cron.NewHelmApplicationStatusUpdateHandlerImpl,
 		wire.Bind(new(cron.HelmApplicationStatusUpdateHandler), new(*cron.HelmApplicationStatusUpdateHandlerImpl)),
+
+		pipeline.NewPresetContainerRegistryHandlerImpl,
+		wire.Bind(new(pipeline.PresetContainerRegistryHandler), new(*pipeline.PresetContainerRegistryHandlerImpl)),
 	)
 	return &App{}, nil
 }

api/restHandler/DockerRegRestHandler.go

@@ -19,9 +19,11 @@ package restHandler
 
 import (
 	"encoding/json"
+	"errors"
 	"github.com/devtron-labs/devtron/api/restHandler/common"
 	delete2 "github.com/devtron-labs/devtron/pkg/delete"
 	"github.com/devtron-labs/devtron/pkg/user/casbin"
+	util2 "github.com/devtron-labs/devtron/util"
 	"net/http"
 	"strings"
 
@@ -165,6 +167,8 @@ func (impl DockerRegRestHandlerImpl) FetchAllDockerAccounts(w http.ResponseWrite
 	}
 	//RBAC enforcer Ends
 
+	result = modifyResponseForPresetRegistry(result)
+
 	common.WriteJsonResp(w, err, result, http.StatusOK)
 }
 
@@ -186,7 +190,9 @@ func (impl DockerRegRestHandlerImpl) FetchOneDockerAccounts(w http.ResponseWrite
 	}
 	//RBAC enforcer Ends
 
-	common.WriteJsonResp(w, err, res, http.StatusOK)
+	registries := []pipeline.DockerArtifactStoreBean{*res}
+	registries = modifyResponseForPresetRegistry(registries)
+	common.WriteJsonResp(w, err, registries[0], http.StatusOK)
 }
 
 func (impl DockerRegRestHandlerImpl) UpdateDockerRegistryConfig(w http.ResponseWriter, r *http.Request) {
@@ -217,6 +223,13 @@ func (impl DockerRegRestHandlerImpl) UpdateDockerRegistryConfig(w http.ResponseW
 			return
 		}
 
+		if bean.Id == util2.DockerPresetContainerRegistryId {
+			impl.logger.Errorw("error in updating devtron preset container registry", "registry", bean.Id)
+			err = errors.New("preset registry can't be updated")
+			common.WriteJsonResp(w, err, nil, http.StatusBadRequest)
+			return
+		}
+
 		// RBAC enforcer applying
 		token := r.Header.Get("token")
 		if ok := impl.enforcer.Enforce(token, casbin.ResourceDocker, casbin.ActionUpdate, strings.ToLower(bean.Id)); !ok {
@@ -245,6 +258,7 @@ func (impl DockerRegRestHandlerImpl) FetchAllDockerRegistryForAutocomplete(w htt
 		return
 	}
 
+	res = modifyResponseForPresetRegistry(res)
 	common.WriteJsonResp(w, err, res, http.StatusOK)
 }
 
@@ -301,3 +315,22 @@ func (impl DockerRegRestHandlerImpl) DeleteDockerRegistryConfig(w http.ResponseW
 	}
 	common.WriteJsonResp(w, err, REG_DELETE_SUCCESS_RESP, http.StatusOK)
 }
+
+func modifyResponseForPresetRegistry(response []pipeline.DockerArtifactStoreBean) []pipeline.DockerArtifactStoreBean {
+	var modifiedResponse []pipeline.DockerArtifactStoreBean
+	for _, bean := range response {
+		if bean.Id == util2.DockerPresetContainerRegistryId {
+			modifiedBean := pipeline.DockerArtifactStoreBean{
+				Id:           bean.Id,
+				RegistryURL:  bean.RegistryURL,
+				RegistryType: bean.RegistryType,
+				Active:       bean.Active,
+				IsDefault:    bean.IsDefault,
+			}
+			modifiedResponse = append(modifiedResponse, modifiedBean)
+		} else {
+			modifiedResponse = append(modifiedResponse, bean)
+		}
+	}
+	return modifiedResponse
+}

internal/sql/repository/AttributesRepository.go

@@ -35,6 +35,7 @@ type AttributesRepository interface {
 	Save(model *Attributes, tx *pg.Tx) (*Attributes, error)
 	Update(model *Attributes, tx *pg.Tx) error
 	FindByKey(key string) (*Attributes, error)
+	FindByKeys(keys []string) ([]*Attributes, error)
 	FindById(id int) (*Attributes, error)
 	FindActiveList() ([]*Attributes, error)
 	GetConnection() (dbConnection *pg.DB)
@@ -69,6 +70,13 @@ func (repo AttributesRepositoryImpl) FindByKey(key string) (*Attributes, error)
 	return model, err
 }
 
+func (repo AttributesRepositoryImpl) FindByKeys(keys []string) ([]*Attributes, error) {
+	var models []*Attributes
+	err := repo.dbConnection.Model(&models).Where("key IN (?)", pg.In(keys)).Where("active = ?", true).
+		Select()
+	return models, err
+}
+
 func (repo AttributesRepositoryImpl) FindById(id int) (*Attributes, error) {
 	model := &Attributes{}
 	err := repo.dbConnection.Model(model).Where("id = ?", id).Where("active = ?", true).

internal/sql/repository/CiArtifactRepository.go

@@ -115,7 +115,7 @@ func (impl CiArtifactRepositoryImpl) GetArtifactsByCDPipeline(cdPipelineId, limi
 	var artifactsAB []CiArtifact
 
 	queryFetchArtifacts := ""
-	queryFetchArtifacts = "SELECT cia.id, cia.data_source, cia.image, cia.image_digest, cia.scan_enabled, cia.scanned FROM ci_artifact cia" +
+	queryFetchArtifacts = "SELECT cia.id, cia.data_source, cia.image, cia.image_digest, cia.scan_enabled, cia.scanned, cia.created_on FROM ci_artifact cia" +
 		" INNER JOIN ci_pipeline cp on cp.id=cia.pipeline_id" +
 		" INNER JOIN pipeline p on p.ci_pipeline_id = cp.id" +
 		" WHERE p.id= ? ORDER BY cia.id DESC"

pkg/attributes/AttributesService.go

@@ -30,10 +30,11 @@ type AttributesService interface {
 	GetById(id int) (*AttributesDto, error)
 	GetActiveList() ([]*AttributesDto, error)
 	GetByKey(key string) (*AttributesDto, error)
+	GetByKeys(keys []string) ([]*AttributesDto, error)
 }
 
 const (
-	HostUrlKey string = "url"
+	HostUrlKey     string = "url"
 	API_SECRET_KEY string = "apiTokenSecret"
 )
 
@@ -196,3 +197,25 @@ func (impl AttributesServiceImpl) GetByKey(key string) (*AttributesDto, error) {
 	}
 	return dto, nil
 }
+
+func (impl AttributesServiceImpl) GetByKeys(keys []string) ([]*AttributesDto, error) {
+	results := make([]*AttributesDto, 0)
+	models, err := impl.attributesRepository.FindByKeys(keys)
+	if err != nil && err != pg.ErrNoRows {
+		impl.logger.Errorw("error in fetching attributes", "error", err, "keys", keys)
+		return nil, err
+	}
+	if err == pg.ErrNoRows {
+		return results, nil
+	}
+	for _, model := range models {
+		dto := &AttributesDto{
+			Id:     model.Id,
+			Active: model.Active,
+			Key:    model.Key,
+			Value:  model.Value,
+		}
+		results = append(results, dto)
+	}
+	return results, nil
+}

pkg/bean/app.go

@@ -531,6 +531,8 @@ type CiArtifactBean struct {
 	IsVulnerable                  bool            `json:"vulnerable,notnull"`
 	ScanEnabled                   bool            `json:"scanEnabled,notnull"`
 	Scanned                       bool            `json:"scanned,notnull"`
+	BuildUsingPresetRegistry      bool            `json:"buildUsingPresetRegistry,omitempty"`
+	PresetImageDeleted            bool            `json:"presetImageDeleted,omitempty"`
 }
 
 type CiArtifactResponse struct {

pkg/pipeline/CiService.go

@@ -26,6 +26,7 @@ import (
 	"github.com/devtron-labs/devtron/pkg/user"
 	"path/filepath"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1"
@@ -34,6 +35,7 @@ import (
 	"github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig"
 	"github.com/devtron-labs/devtron/internal/util"
 	"github.com/devtron-labs/devtron/pkg/bean"
+	util3 "github.com/devtron-labs/devtron/util"
 	util2 "github.com/devtron-labs/devtron/util/event"
 	"go.uber.org/zap"
 )
@@ -56,6 +58,7 @@ type CiServiceImpl struct {
 	prePostCiScriptHistoryService history.PrePostCiScriptHistoryService
 	pipelineStageService          PipelineStageService
 	userService                   user.UserService
+	presetRegistryHandler         PresetContainerRegistryHandler
 }
 
 func NewCiServiceImpl(Logger *zap.SugaredLogger, workflowService WorkflowService,
@@ -64,7 +67,7 @@ func NewCiServiceImpl(Logger *zap.SugaredLogger, workflowService WorkflowService
 	eventFactory client.EventFactory, mergeUtil *util.MergeUtil, ciPipelineRepository pipelineConfig.CiPipelineRepository,
 	prePostCiScriptHistoryService history.PrePostCiScriptHistoryService,
 	pipelineStageService PipelineStageService,
-	userService user.UserService) *CiServiceImpl {
+	userService user.UserService, presetContainerRegistryHandler PresetContainerRegistryHandler) *CiServiceImpl {
 	return &CiServiceImpl{
 		Logger:                        Logger,
 		workflowService:               workflowService,
@@ -78,6 +81,7 @@ func NewCiServiceImpl(Logger *zap.SugaredLogger, workflowService WorkflowService
 		prePostCiScriptHistoryService: prePostCiScriptHistoryService,
 		pipelineStageService:          pipelineStageService,
 		userService:                   userService,
+		presetRegistryHandler:         presetContainerRegistryHandler,
 	}
 }
 
@@ -385,15 +389,27 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig.
 		return nil, err
 	}
 	dockerfilePath := filepath.Join(pipeline.CiTemplate.GitMaterial.CheckoutPath, pipeline.CiTemplate.DockerfilePath)
+
+	dockerRepository := pipeline.CiTemplate.DockerRepository
+	dockerRegistryId := pipeline.CiTemplate.DockerRegistry.Id
+	registryUrl := pipeline.CiTemplate.DockerRegistry.RegistryURL
+	if dockerRegistryId == util3.DockerPresetContainerRegistryId {
+		registryConfigBean := impl.presetRegistryHandler.GetPresetDockerRegistryConfigBean()
+		dockerRepository = registryConfigBean.PresetRegistryRepoName
+		if isPublicRegistry(registryConfigBean, registryUrl) {
+			dockerRepository = getPublicRegistryRepoName(dockerRepository, dockerImageTag)
+			dockerImageTag = registryConfigBean.PresetPublicRegistryImgTagValue
+		}
+	}
 	workflowRequest := &WorkflowRequest{
 		WorkflowNamePrefix:         strconv.Itoa(savedWf.Id) + "-" + savedWf.Name,
 		PipelineName:               pipeline.Name,
 		PipelineId:                 pipeline.Id,
-		DockerRegistryId:           pipeline.CiTemplate.DockerRegistry.Id,
+		DockerRegistryId:           dockerRegistryId,
 		DockerRegistryType:         string(pipeline.CiTemplate.DockerRegistry.RegistryType),
 		DockerImageTag:             dockerImageTag,
-		DockerRegistryURL:          pipeline.CiTemplate.DockerRegistry.RegistryURL,
-		DockerRepository:           pipeline.CiTemplate.DockerRepository,
+		DockerRegistryURL:          registryUrl,
+		DockerRepository:           dockerRepository,
 		DockerBuildArgs:            string(merged),
 		DockerBuildTargetPlatform:  pipeline.CiTemplate.TargetPlatform,
 		DockerFileLocation:         dockerfilePath,
@@ -450,6 +466,15 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig.
 	return workflowRequest, nil
 }
 
+func isPublicRegistry(configBean *PresetDockerRegistryConfigBean, url string) bool {
+	publicRegistry := configBean.PresetPublicRegistry
+	return strings.Index(url, publicRegistry) == 0
+}
+
+func getPublicRegistryRepoName(dockerRepository string, dockerImgTag string) string {
+	return dockerRepository + "-" + dockerImgTag
+}
+
 func buildCiStepsDataFromDockerBuildScripts(dockerBuildScripts []*bean.CiScript) []*bean2.StepObject {
 	//before plugin support, few variables were set as env vars in ci-runner
 	//these variables are now moved to global vars in plugin steps, but to avoid error in old scripts adding those variables in payload

pkg/pipeline/DockerRegistryConfig.go

@@ -18,8 +18,10 @@
 package pipeline
 
 import (
+	"errors"
 	"fmt"
 	"github.com/devtron-labs/devtron/pkg/sql"
+	util2 "github.com/devtron-labs/devtron/util"
 	"time"
 
 	"github.com/devtron-labs/devtron/internal/constants"
@@ -243,6 +245,10 @@ func (impl DockerRegistryConfigImpl) Delete(storeId string) (string, error) {
 }
 
 func (impl DockerRegistryConfigImpl) DeleteReg(bean *DockerArtifactStoreBean) error {
+	if bean.Id == util2.DockerPresetContainerRegistryId {
+		impl.logger.Errorw("error in deleting devtron preset container registry", "registry", bean.Id)
+		return errors.New("preset registry can't be deleted")
+	}
 	dockerReg, err := impl.dockerArtifactStoreRepository.FindOne(bean.Id)
 	if err != nil {
 		impl.logger.Errorw("No matching entry found for delete.", "id", bean.Id, "err", err)

pkg/pipeline/PipelineBuilder.go

@@ -30,6 +30,7 @@ import (
 	repository4 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository"
 	"github.com/devtron-labs/devtron/pkg/sql"
 	util3 "github.com/devtron-labs/devtron/pkg/util"
+	util1 "github.com/devtron-labs/devtron/util"
 	"net/http"
 	"net/url"
 	"sort"
@@ -145,6 +146,7 @@ type PipelineBuilderImpl struct {
 	chartService                     chart.ChartService
 	helmAppService                   client.HelmAppService
 	deploymentGroupRepository        repository.DeploymentGroupRepository
+	presetRegistryHandler            PresetContainerRegistryHandler
 }
 
 func NewPipelineBuilderImpl(logger *zap.SugaredLogger,
@@ -180,7 +182,7 @@ func NewPipelineBuilderImpl(logger *zap.SugaredLogger,
 	pipelineStageService PipelineStageService, chartRefRepository chartRepoRepository.ChartRefRepository,
 	chartTemplateService util.ChartTemplateService, chartService chart.ChartService,
 	helmAppService client.HelmAppService,
-	deploymentGroupRepository repository.DeploymentGroupRepository) *PipelineBuilderImpl {
+	deploymentGroupRepository repository.DeploymentGroupRepository, presetRegistryHandler PresetContainerRegistryHandler) *PipelineBuilderImpl {
 	return &PipelineBuilderImpl{
 		logger:                           logger,
 		dbPipelineOrchestrator:           dbPipelineOrchestrator,
@@ -220,6 +222,7 @@ func NewPipelineBuilderImpl(logger *zap.SugaredLogger,
 		chartService:                     chartService,
 		helmAppService:                   helmAppService,
 		deploymentGroupRepository:        deploymentGroupRepository,
+		presetRegistryHandler:            presetRegistryHandler,
 	}
 }
 
@@ -391,12 +394,17 @@ func (impl PipelineBuilderImpl) getCiTemplateVariables(appId int) (ciConfig *bea
 		impl.logger.Errorw("invalid reg url", "err", err)
 		return nil, err
 	}
+	dockerRepository := template.DockerRepository
+	dockerRegistryId := template.DockerRegistry.Id
+	if dockerRegistryId == util1.DockerPresetContainerRegistryId {
+		dockerRepository = impl.presetRegistryHandler.GetPresetDockerRegistryConfigBean().PresetRegistryRepoName
+	}
 	ciConfig = &bean.CiConfigRequest{
 		Id:                template.Id,
 		AppId:             template.AppId,
 		AppName:           template.App.AppName,
-		DockerRepository:  template.DockerRepository,
-		DockerRegistry:    template.DockerRegistry.Id,
+		DockerRepository:  dockerRepository,
+		DockerRegistry:    dockerRegistryId,
 		DockerRegistryUrl: regHost,
 		DockerBuildConfig: &bean.DockerBuildConfig{DockerfilePath: template.DockerfilePath, Args: dockerArgs, GitMaterialId: template.GitMaterialId, TargetPlatform: template.TargetPlatform},
 		Version:           template.Version,
@@ -1761,6 +1769,7 @@ func (impl PipelineBuilderImpl) BuildArtifactsForParentStage(cdPipelineId int, p
 func (impl PipelineBuilderImpl) BuildArtifactsForCdStage(pipelineId int, stageType bean2.WorkflowType, ciArtifacts []bean.CiArtifactBean, artifactMap map[int]int, parent bool, limit int, parentCdId int) ([]bean.CiArtifactBean, map[int]int, error) {
 	//getting running artifact id for parent cd
 	parentCdRunningArtifactId := 0
+	presetDockerRegistryConfigBean := impl.presetRegistryHandler.GetPresetDockerRegistryConfigBean()
 	if parentCdId > 0 && parent {
 		parentCdWfrList, err := impl.cdWorkflowRepository.FindArtifactByPipelineIdAndRunnerType(parentCdId, bean2.CD_WORKFLOW_TYPE_DEPLOY, 1)
 		if err != nil {
@@ -1810,6 +1819,7 @@ func (impl PipelineBuilderImpl) BuildArtifactsForCdStage(pipelineId int, stageTy
 				if runningOnParentCd {
 					ciArtifact.RunningOnParentCd = runningOnParentCd
 				}
+				impl.checkAndUpdatePresetRegistryData(&ciArtifact, presetDockerRegistryConfigBean, wfr.CdWorkflow.CiArtifact.CreatedOn)
 				ciArtifacts = append(ciArtifacts, ciArtifact)
 				//storing index of ci artifact for using when updating old entry
 				artifactMap[wfr.CdWorkflow.CiArtifact.Id] = len(ciArtifacts) - 1
@@ -1835,26 +1845,42 @@ func (impl PipelineBuilderImpl) BuildArtifactsForCIParent(cdPipelineId int, ciAr
 		impl.logger.Errorw("error in getting artifacts for ci", "err", err)
 		return ciArtifacts, err
 	}
+	presetDockerRegistryConfigBean := impl.presetRegistryHandler.GetPresetDockerRegistryConfigBean()
 	for _, artifact := range artifacts {
 		if _, ok := artifactMap[artifact.Id]; !ok {
 			mInfo, err := parseMaterialInfo([]byte(artifact.MaterialInfo), artifact.DataSource)
 			if err != nil {
 				mInfo = []byte("[]")
 				impl.logger.Errorw("Error in parsing artifact material info", "err", err, "artifact", artifact)
 			}
-			ciArtifacts = append(ciArtifacts, bean.CiArtifactBean{
+			artifactBean := bean.CiArtifactBean{
 				Id:           artifact.Id,
 				Image:        artifact.Image,
 				ImageDigest:  artifact.ImageDigest,
 				MaterialInfo: mInfo,
 				ScanEnabled:  artifact.ScanEnabled,
 				Scanned:      artifact.Scanned,
-			})
+			}
+			impl.checkAndUpdatePresetRegistryData(&artifactBean, presetDockerRegistryConfigBean, artifact.CreatedOn)
+			ciArtifacts = append(ciArtifacts, artifactBean)
 		}
 	}
 	return ciArtifacts, nil
 }
 
+func (impl PipelineBuilderImpl) checkAndUpdatePresetRegistryData(artifactBean *bean.CiArtifactBean, presetDockerRegistryConfigBean *PresetDockerRegistryConfigBean, createdOn time.Time) {
+	dockerImage := artifactBean.Image
+	presetRegistryRepoName := presetDockerRegistryConfigBean.PresetRegistryRepoName
+	buildUsingPresetRegistry := strings.Contains(dockerImage, presetRegistryRepoName)
+	artifactBean.BuildUsingPresetRegistry = buildUsingPresetRegistry
+	if buildUsingPresetRegistry {
+		imageCreatedTime := createdOn
+		timegapDurationSinceBuild := time.Since(imageCreatedTime)
+		presetExpiryCnfigrdDuration := time.Duration(presetDockerRegistryConfigBean.PresetRegistryImageExpiryTimeInSecs) * time.Second
+		artifactBean.PresetImageDeleted = timegapDurationSinceBuild > presetExpiryCnfigrdDuration
+	}
+}
+
 func (impl PipelineBuilderImpl) FetchArtifactForRollback(cdPipelineId int) (bean.CiArtifactResponse, error) {
 	var ciArtifacts []bean.CiArtifactBean
 	var ciArtifactsResponse bean.CiArtifactResponse