diff --git a/Wire.go b/Wire.go index 8c9f9a018f..2c835cd3b4 100644 --- a/Wire.go +++ b/Wire.go @@ -90,6 +90,7 @@ import ( jira2 "github.com/devtron-labs/devtron/pkg/jira" "github.com/devtron-labs/devtron/pkg/notifier" "github.com/devtron-labs/devtron/pkg/pipeline" + //pipelineCron "github.com/devtron-labs/devtron/pkg/pipeline/cron" history3 "github.com/devtron-labs/devtron/pkg/pipeline/history" repository3 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" repository5 "github.com/devtron-labs/devtron/pkg/pipeline/repository" @@ -731,6 +732,9 @@ func InitializeApp() (*App, error) { // AuthWireSet, cron.NewHelmApplicationStatusUpdateHandlerImpl, wire.Bind(new(cron.HelmApplicationStatusUpdateHandler), new(*cron.HelmApplicationStatusUpdateHandlerImpl)), + + pipeline.NewPresetContainerRegistryHandlerImpl, + wire.Bind(new(pipeline.PresetContainerRegistryHandler), new(*pipeline.PresetContainerRegistryHandlerImpl)), ) return &App{}, nil } diff --git a/api/restHandler/DockerRegRestHandler.go b/api/restHandler/DockerRegRestHandler.go index a3780e3409..ae00041e9a 100644 --- a/api/restHandler/DockerRegRestHandler.go +++ b/api/restHandler/DockerRegRestHandler.go @@ -19,9 +19,11 @@ package restHandler import ( "encoding/json" + "errors" "github.com/devtron-labs/devtron/api/restHandler/common" delete2 "github.com/devtron-labs/devtron/pkg/delete" "github.com/devtron-labs/devtron/pkg/user/casbin" + util2 "github.com/devtron-labs/devtron/util" "net/http" "strings" @@ -165,6 +167,8 @@ func (impl DockerRegRestHandlerImpl) FetchAllDockerAccounts(w http.ResponseWrite } //RBAC enforcer Ends + result = modifyResponseForPresetRegistry(result) + common.WriteJsonResp(w, err, result, http.StatusOK) } @@ -186,7 +190,9 @@ func (impl DockerRegRestHandlerImpl) FetchOneDockerAccounts(w http.ResponseWrite } //RBAC enforcer Ends - common.WriteJsonResp(w, err, res, http.StatusOK) + registries := []pipeline.DockerArtifactStoreBean{*res} + registries = modifyResponseForPresetRegistry(registries) + common.WriteJsonResp(w, err, registries[0], http.StatusOK) } func (impl DockerRegRestHandlerImpl) UpdateDockerRegistryConfig(w http.ResponseWriter, r *http.Request) { @@ -217,6 +223,13 @@ func (impl DockerRegRestHandlerImpl) UpdateDockerRegistryConfig(w http.ResponseW return } + if bean.Id == util2.DockerPresetContainerRegistryId { + impl.logger.Errorw("error in updating devtron preset container registry", "registry", bean.Id) + err = errors.New("preset registry can't be updated") + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + return + } + // RBAC enforcer applying token := r.Header.Get("token") if ok := impl.enforcer.Enforce(token, casbin.ResourceDocker, casbin.ActionUpdate, strings.ToLower(bean.Id)); !ok { @@ -245,6 +258,7 @@ func (impl DockerRegRestHandlerImpl) FetchAllDockerRegistryForAutocomplete(w htt return } + res = modifyResponseForPresetRegistry(res) common.WriteJsonResp(w, err, res, http.StatusOK) } @@ -301,3 +315,22 @@ func (impl DockerRegRestHandlerImpl) DeleteDockerRegistryConfig(w http.ResponseW } common.WriteJsonResp(w, err, REG_DELETE_SUCCESS_RESP, http.StatusOK) } + +func modifyResponseForPresetRegistry(response []pipeline.DockerArtifactStoreBean) []pipeline.DockerArtifactStoreBean { + var modifiedResponse []pipeline.DockerArtifactStoreBean + for _, bean := range response { + if bean.Id == util2.DockerPresetContainerRegistryId { + modifiedBean := pipeline.DockerArtifactStoreBean{ + Id: bean.Id, + RegistryURL: bean.RegistryURL, + RegistryType: bean.RegistryType, + Active: bean.Active, + IsDefault: bean.IsDefault, + } + modifiedResponse = append(modifiedResponse, modifiedBean) + } else { + modifiedResponse = append(modifiedResponse, bean) + } + } + return modifiedResponse +} diff --git a/internal/sql/repository/AttributesRepository.go b/internal/sql/repository/AttributesRepository.go index e8a8ec90c1..de8bdd718c 100644 --- a/internal/sql/repository/AttributesRepository.go +++ b/internal/sql/repository/AttributesRepository.go @@ -35,6 +35,7 @@ type AttributesRepository interface { Save(model *Attributes, tx *pg.Tx) (*Attributes, error) Update(model *Attributes, tx *pg.Tx) error FindByKey(key string) (*Attributes, error) + FindByKeys(keys []string) ([]*Attributes, error) FindById(id int) (*Attributes, error) FindActiveList() ([]*Attributes, error) GetConnection() (dbConnection *pg.DB) @@ -69,6 +70,13 @@ func (repo AttributesRepositoryImpl) FindByKey(key string) (*Attributes, error) return model, err } +func (repo AttributesRepositoryImpl) FindByKeys(keys []string) ([]*Attributes, error) { + var models []*Attributes + err := repo.dbConnection.Model(&models).Where("key IN (?)", pg.In(keys)).Where("active = ?", true). + Select() + return models, err +} + func (repo AttributesRepositoryImpl) FindById(id int) (*Attributes, error) { model := &Attributes{} err := repo.dbConnection.Model(model).Where("id = ?", id).Where("active = ?", true). diff --git a/internal/sql/repository/CiArtifactRepository.go b/internal/sql/repository/CiArtifactRepository.go index 6698b045c0..79230a39ca 100644 --- a/internal/sql/repository/CiArtifactRepository.go +++ b/internal/sql/repository/CiArtifactRepository.go @@ -115,7 +115,7 @@ func (impl CiArtifactRepositoryImpl) GetArtifactsByCDPipeline(cdPipelineId, limi var artifactsAB []CiArtifact queryFetchArtifacts := "" - queryFetchArtifacts = "SELECT cia.id, cia.data_source, cia.image, cia.image_digest, cia.scan_enabled, cia.scanned FROM ci_artifact cia" + + queryFetchArtifacts = "SELECT cia.id, cia.data_source, cia.image, cia.image_digest, cia.scan_enabled, cia.scanned, cia.created_on FROM ci_artifact cia" + " INNER JOIN ci_pipeline cp on cp.id=cia.pipeline_id" + " INNER JOIN pipeline p on p.ci_pipeline_id = cp.id" + " WHERE p.id= ? ORDER BY cia.id DESC" diff --git a/pkg/attributes/AttributesService.go b/pkg/attributes/AttributesService.go index 1e68cc03b0..dcab55c67f 100644 --- a/pkg/attributes/AttributesService.go +++ b/pkg/attributes/AttributesService.go @@ -30,10 +30,11 @@ type AttributesService interface { GetById(id int) (*AttributesDto, error) GetActiveList() ([]*AttributesDto, error) GetByKey(key string) (*AttributesDto, error) + GetByKeys(keys []string) ([]*AttributesDto, error) } const ( - HostUrlKey string = "url" + HostUrlKey string = "url" API_SECRET_KEY string = "apiTokenSecret" ) @@ -196,3 +197,25 @@ func (impl AttributesServiceImpl) GetByKey(key string) (*AttributesDto, error) { } return dto, nil } + +func (impl AttributesServiceImpl) GetByKeys(keys []string) ([]*AttributesDto, error) { + results := make([]*AttributesDto, 0) + models, err := impl.attributesRepository.FindByKeys(keys) + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error in fetching attributes", "error", err, "keys", keys) + return nil, err + } + if err == pg.ErrNoRows { + return results, nil + } + for _, model := range models { + dto := &AttributesDto{ + Id: model.Id, + Active: model.Active, + Key: model.Key, + Value: model.Value, + } + results = append(results, dto) + } + return results, nil +} diff --git a/pkg/bean/app.go b/pkg/bean/app.go index f02f74fd86..de91041772 100644 --- a/pkg/bean/app.go +++ b/pkg/bean/app.go @@ -531,6 +531,8 @@ type CiArtifactBean struct { IsVulnerable bool `json:"vulnerable,notnull"` ScanEnabled bool `json:"scanEnabled,notnull"` Scanned bool `json:"scanned,notnull"` + BuildUsingPresetRegistry bool `json:"buildUsingPresetRegistry,omitempty"` + PresetImageDeleted bool `json:"presetImageDeleted,omitempty"` } type CiArtifactResponse struct { diff --git a/pkg/pipeline/CiService.go b/pkg/pipeline/CiService.go index b4d539bd26..2f4b44b87f 100644 --- a/pkg/pipeline/CiService.go +++ b/pkg/pipeline/CiService.go @@ -26,6 +26,7 @@ import ( "github.com/devtron-labs/devtron/pkg/user" "path/filepath" "strconv" + "strings" "time" "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" @@ -34,6 +35,7 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/bean" + util3 "github.com/devtron-labs/devtron/util" util2 "github.com/devtron-labs/devtron/util/event" "go.uber.org/zap" ) @@ -56,6 +58,7 @@ type CiServiceImpl struct { prePostCiScriptHistoryService history.PrePostCiScriptHistoryService pipelineStageService PipelineStageService userService user.UserService + presetRegistryHandler PresetContainerRegistryHandler } func NewCiServiceImpl(Logger *zap.SugaredLogger, workflowService WorkflowService, @@ -64,7 +67,7 @@ func NewCiServiceImpl(Logger *zap.SugaredLogger, workflowService WorkflowService eventFactory client.EventFactory, mergeUtil *util.MergeUtil, ciPipelineRepository pipelineConfig.CiPipelineRepository, prePostCiScriptHistoryService history.PrePostCiScriptHistoryService, pipelineStageService PipelineStageService, - userService user.UserService) *CiServiceImpl { + userService user.UserService, presetContainerRegistryHandler PresetContainerRegistryHandler) *CiServiceImpl { return &CiServiceImpl{ Logger: Logger, workflowService: workflowService, @@ -78,6 +81,7 @@ func NewCiServiceImpl(Logger *zap.SugaredLogger, workflowService WorkflowService prePostCiScriptHistoryService: prePostCiScriptHistoryService, pipelineStageService: pipelineStageService, userService: userService, + presetRegistryHandler: presetContainerRegistryHandler, } } @@ -385,15 +389,27 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. return nil, err } dockerfilePath := filepath.Join(pipeline.CiTemplate.GitMaterial.CheckoutPath, pipeline.CiTemplate.DockerfilePath) + + dockerRepository := pipeline.CiTemplate.DockerRepository + dockerRegistryId := pipeline.CiTemplate.DockerRegistry.Id + registryUrl := pipeline.CiTemplate.DockerRegistry.RegistryURL + if dockerRegistryId == util3.DockerPresetContainerRegistryId { + registryConfigBean := impl.presetRegistryHandler.GetPresetDockerRegistryConfigBean() + dockerRepository = registryConfigBean.PresetRegistryRepoName + if isPublicRegistry(registryConfigBean, registryUrl) { + dockerRepository = getPublicRegistryRepoName(dockerRepository, dockerImageTag) + dockerImageTag = registryConfigBean.PresetPublicRegistryImgTagValue + } + } workflowRequest := &WorkflowRequest{ WorkflowNamePrefix: strconv.Itoa(savedWf.Id) + "-" + savedWf.Name, PipelineName: pipeline.Name, PipelineId: pipeline.Id, - DockerRegistryId: pipeline.CiTemplate.DockerRegistry.Id, + DockerRegistryId: dockerRegistryId, DockerRegistryType: string(pipeline.CiTemplate.DockerRegistry.RegistryType), DockerImageTag: dockerImageTag, - DockerRegistryURL: pipeline.CiTemplate.DockerRegistry.RegistryURL, - DockerRepository: pipeline.CiTemplate.DockerRepository, + DockerRegistryURL: registryUrl, + DockerRepository: dockerRepository, DockerBuildArgs: string(merged), DockerBuildTargetPlatform: pipeline.CiTemplate.TargetPlatform, DockerFileLocation: dockerfilePath, @@ -450,6 +466,15 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. return workflowRequest, nil } +func isPublicRegistry(configBean *PresetDockerRegistryConfigBean, url string) bool { + publicRegistry := configBean.PresetPublicRegistry + return strings.Index(url, publicRegistry) == 0 +} + +func getPublicRegistryRepoName(dockerRepository string, dockerImgTag string) string { + return dockerRepository + "-" + dockerImgTag +} + func buildCiStepsDataFromDockerBuildScripts(dockerBuildScripts []*bean.CiScript) []*bean2.StepObject { //before plugin support, few variables were set as env vars in ci-runner //these variables are now moved to global vars in plugin steps, but to avoid error in old scripts adding those variables in payload diff --git a/pkg/pipeline/DockerRegistryConfig.go b/pkg/pipeline/DockerRegistryConfig.go index c56da10916..3e9ed62444 100644 --- a/pkg/pipeline/DockerRegistryConfig.go +++ b/pkg/pipeline/DockerRegistryConfig.go @@ -18,8 +18,10 @@ package pipeline import ( + "errors" "fmt" "github.com/devtron-labs/devtron/pkg/sql" + util2 "github.com/devtron-labs/devtron/util" "time" "github.com/devtron-labs/devtron/internal/constants" @@ -243,6 +245,10 @@ func (impl DockerRegistryConfigImpl) Delete(storeId string) (string, error) { } func (impl DockerRegistryConfigImpl) DeleteReg(bean *DockerArtifactStoreBean) error { + if bean.Id == util2.DockerPresetContainerRegistryId { + impl.logger.Errorw("error in deleting devtron preset container registry", "registry", bean.Id) + return errors.New("preset registry can't be deleted") + } dockerReg, err := impl.dockerArtifactStoreRepository.FindOne(bean.Id) if err != nil { impl.logger.Errorw("No matching entry found for delete.", "id", bean.Id, "err", err) diff --git a/pkg/pipeline/PipelineBuilder.go b/pkg/pipeline/PipelineBuilder.go index d1c09b7d4f..7c25dbb9a9 100644 --- a/pkg/pipeline/PipelineBuilder.go +++ b/pkg/pipeline/PipelineBuilder.go @@ -30,6 +30,7 @@ import ( repository4 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" "github.com/devtron-labs/devtron/pkg/sql" util3 "github.com/devtron-labs/devtron/pkg/util" + util1 "github.com/devtron-labs/devtron/util" "net/http" "net/url" "sort" @@ -145,6 +146,7 @@ type PipelineBuilderImpl struct { chartService chart.ChartService helmAppService client.HelmAppService deploymentGroupRepository repository.DeploymentGroupRepository + presetRegistryHandler PresetContainerRegistryHandler } func NewPipelineBuilderImpl(logger *zap.SugaredLogger, @@ -180,7 +182,7 @@ func NewPipelineBuilderImpl(logger *zap.SugaredLogger, pipelineStageService PipelineStageService, chartRefRepository chartRepoRepository.ChartRefRepository, chartTemplateService util.ChartTemplateService, chartService chart.ChartService, helmAppService client.HelmAppService, - deploymentGroupRepository repository.DeploymentGroupRepository) *PipelineBuilderImpl { + deploymentGroupRepository repository.DeploymentGroupRepository, presetRegistryHandler PresetContainerRegistryHandler) *PipelineBuilderImpl { return &PipelineBuilderImpl{ logger: logger, dbPipelineOrchestrator: dbPipelineOrchestrator, @@ -220,6 +222,7 @@ func NewPipelineBuilderImpl(logger *zap.SugaredLogger, chartService: chartService, helmAppService: helmAppService, deploymentGroupRepository: deploymentGroupRepository, + presetRegistryHandler: presetRegistryHandler, } } @@ -391,12 +394,17 @@ func (impl PipelineBuilderImpl) getCiTemplateVariables(appId int) (ciConfig *bea impl.logger.Errorw("invalid reg url", "err", err) return nil, err } + dockerRepository := template.DockerRepository + dockerRegistryId := template.DockerRegistry.Id + if dockerRegistryId == util1.DockerPresetContainerRegistryId { + dockerRepository = impl.presetRegistryHandler.GetPresetDockerRegistryConfigBean().PresetRegistryRepoName + } ciConfig = &bean.CiConfigRequest{ Id: template.Id, AppId: template.AppId, AppName: template.App.AppName, - DockerRepository: template.DockerRepository, - DockerRegistry: template.DockerRegistry.Id, + DockerRepository: dockerRepository, + DockerRegistry: dockerRegistryId, DockerRegistryUrl: regHost, DockerBuildConfig: &bean.DockerBuildConfig{DockerfilePath: template.DockerfilePath, Args: dockerArgs, GitMaterialId: template.GitMaterialId, TargetPlatform: template.TargetPlatform}, Version: template.Version, @@ -1761,6 +1769,7 @@ func (impl PipelineBuilderImpl) BuildArtifactsForParentStage(cdPipelineId int, p func (impl PipelineBuilderImpl) BuildArtifactsForCdStage(pipelineId int, stageType bean2.WorkflowType, ciArtifacts []bean.CiArtifactBean, artifactMap map[int]int, parent bool, limit int, parentCdId int) ([]bean.CiArtifactBean, map[int]int, error) { //getting running artifact id for parent cd parentCdRunningArtifactId := 0 + presetDockerRegistryConfigBean := impl.presetRegistryHandler.GetPresetDockerRegistryConfigBean() if parentCdId > 0 && parent { parentCdWfrList, err := impl.cdWorkflowRepository.FindArtifactByPipelineIdAndRunnerType(parentCdId, bean2.CD_WORKFLOW_TYPE_DEPLOY, 1) if err != nil { @@ -1810,6 +1819,7 @@ func (impl PipelineBuilderImpl) BuildArtifactsForCdStage(pipelineId int, stageTy if runningOnParentCd { ciArtifact.RunningOnParentCd = runningOnParentCd } + impl.checkAndUpdatePresetRegistryData(&ciArtifact, presetDockerRegistryConfigBean, wfr.CdWorkflow.CiArtifact.CreatedOn) ciArtifacts = append(ciArtifacts, ciArtifact) //storing index of ci artifact for using when updating old entry artifactMap[wfr.CdWorkflow.CiArtifact.Id] = len(ciArtifacts) - 1 @@ -1835,6 +1845,7 @@ func (impl PipelineBuilderImpl) BuildArtifactsForCIParent(cdPipelineId int, ciAr impl.logger.Errorw("error in getting artifacts for ci", "err", err) return ciArtifacts, err } + presetDockerRegistryConfigBean := impl.presetRegistryHandler.GetPresetDockerRegistryConfigBean() for _, artifact := range artifacts { if _, ok := artifactMap[artifact.Id]; !ok { mInfo, err := parseMaterialInfo([]byte(artifact.MaterialInfo), artifact.DataSource) @@ -1842,19 +1853,34 @@ func (impl PipelineBuilderImpl) BuildArtifactsForCIParent(cdPipelineId int, ciAr mInfo = []byte("[]") impl.logger.Errorw("Error in parsing artifact material info", "err", err, "artifact", artifact) } - ciArtifacts = append(ciArtifacts, bean.CiArtifactBean{ + artifactBean := bean.CiArtifactBean{ Id: artifact.Id, Image: artifact.Image, ImageDigest: artifact.ImageDigest, MaterialInfo: mInfo, ScanEnabled: artifact.ScanEnabled, Scanned: artifact.Scanned, - }) + } + impl.checkAndUpdatePresetRegistryData(&artifactBean, presetDockerRegistryConfigBean, artifact.CreatedOn) + ciArtifacts = append(ciArtifacts, artifactBean) } } return ciArtifacts, nil } +func (impl PipelineBuilderImpl) checkAndUpdatePresetRegistryData(artifactBean *bean.CiArtifactBean, presetDockerRegistryConfigBean *PresetDockerRegistryConfigBean, createdOn time.Time) { + dockerImage := artifactBean.Image + presetRegistryRepoName := presetDockerRegistryConfigBean.PresetRegistryRepoName + buildUsingPresetRegistry := strings.Contains(dockerImage, presetRegistryRepoName) + artifactBean.BuildUsingPresetRegistry = buildUsingPresetRegistry + if buildUsingPresetRegistry { + imageCreatedTime := createdOn + timegapDurationSinceBuild := time.Since(imageCreatedTime) + presetExpiryCnfigrdDuration := time.Duration(presetDockerRegistryConfigBean.PresetRegistryImageExpiryTimeInSecs) * time.Second + artifactBean.PresetImageDeleted = timegapDurationSinceBuild > presetExpiryCnfigrdDuration + } +} + func (impl PipelineBuilderImpl) FetchArtifactForRollback(cdPipelineId int) (bean.CiArtifactResponse, error) { var ciArtifacts []bean.CiArtifactBean var ciArtifactsResponse bean.CiArtifactResponse diff --git a/pkg/pipeline/PresetContainerRegistryUpdateHandler.go b/pkg/pipeline/PresetContainerRegistryUpdateHandler.go new file mode 100644 index 0000000000..94c46680f1 --- /dev/null +++ b/pkg/pipeline/PresetContainerRegistryUpdateHandler.go @@ -0,0 +1,242 @@ +package pipeline + +import ( + "encoding/json" + "errors" + "fmt" + "github.com/caarlos0/env" + "github.com/devtron-labs/devtron/pkg/attributes" + util2 "github.com/devtron-labs/devtron/util" + "github.com/robfig/cron/v3" + "go.uber.org/zap" + "log" + "strconv" +) + +type PresetContainerRegistryHandler interface { + SyncAndUpdatePresetContainerRegistry() + GetPresetDockerRegistryConfigBean() *PresetDockerRegistryConfigBean +} + +type PresetContainerRegistryHandlerImpl struct { + logger *zap.SugaredLogger + dockerRegistryConfig DockerRegistryConfig + presetDockerRegistryConfigBean *PresetDockerRegistryConfigBean + cron *cron.Cron + attributeService attributes.AttributesService +} + +const PresetRegistryRepoNameKey = "PresetRegistryRepoName" +const PresetRegistryExpiryTimeKey = "PresetRegistryExpiryTime" + +func NewPresetContainerRegistryHandlerImpl(logger *zap.SugaredLogger, + dockerRegistryConfig DockerRegistryConfig, attributeService attributes.AttributesService) *PresetContainerRegistryHandlerImpl { + cron := cron.New( + cron.WithChain()) + cron.Start() + presetDockerRegistryConfigBean := LoadConfig() + impl := &PresetContainerRegistryHandlerImpl{ + logger: logger, + cron: cron, + dockerRegistryConfig: dockerRegistryConfig, + presetDockerRegistryConfigBean: presetDockerRegistryConfigBean, + attributeService: attributeService, + } + _, err := cron.AddFunc(presetDockerRegistryConfigBean.PresetRegistryUpdateCronExpr, impl.SyncAndUpdatePresetContainerRegistry) + if err != nil { + logger.Errorw("error in starting preset container registry update cron job", "err", err) + return nil + } + go impl.loadExpiryAndRepoNameFromDb() + return impl +} + +type PresetDockerRegistryConfigBean struct { + PresetRegistrySyncUrl string `env:"PRESET_REGISTRY_SYNC_URL" envDefault:"https://api.devtron.ai/presetCR"` + PresetRegistryUpdateCronExpr string `env:"PRESET_REGISTRY_UPDATE_CRON_EXPR" envDefault:"0 */1 * * *"` + PresetPublicRegistryImgTagValue string `env:"PRESET_PUBLIC_REGISTRY_IMG_TAG" envDefault:"24h"` + PresetPublicRegistry string `env:"PRESET_PUBLIC_REGISTRY" envDefault:"ttl.sh"` + PresetRegistryRepoName string + PresetRegistryImageExpiryTimeInSecs int +} + +func LoadConfig() *PresetDockerRegistryConfigBean { + cfg := &PresetDockerRegistryConfigBean{} + err := env.Parse(cfg) + if err != nil { + log.Fatal("error occurred while loading docker registry config ") + } + return cfg +} + +func (impl *PresetContainerRegistryHandlerImpl) GetPresetDockerRegistryConfigBean() *PresetDockerRegistryConfigBean { + + cfg := impl.presetDockerRegistryConfigBean + if cfg.PresetRegistryImageExpiryTimeInSecs == 0 || cfg.PresetRegistryRepoName == "" { + impl.loadExpiryAndRepoNameFromDb() + } + return cfg +} + +func (impl *PresetContainerRegistryHandlerImpl) loadExpiryAndRepoNameFromDb() { + imageExpiryTimeInSecs := 0 + presetRepoName := "" + keys := []string{PresetRegistryExpiryTimeKey, PresetRegistryRepoNameKey} + keyAttributes, err := impl.attributeService.GetByKeys(keys) + if err != nil { + return + } + for _, attribute := range keyAttributes { + attrValue := attribute.Value + attrKey := attribute.Key + if attrKey == PresetRegistryRepoNameKey { + presetRepoName = attrValue + } else if attrKey == PresetRegistryExpiryTimeKey { + imageExpiryTimeInSecs, _ = strconv.Atoi(attrValue) + } + } + registryConfigBean := impl.presetDockerRegistryConfigBean + if registryConfigBean.PresetRegistryImageExpiryTimeInSecs == 0 { + registryConfigBean.PresetRegistryImageExpiryTimeInSecs = imageExpiryTimeInSecs + } + if registryConfigBean.PresetRegistryRepoName == "" { + registryConfigBean.PresetRegistryRepoName = presetRepoName + } +} + +func (impl *PresetContainerRegistryHandlerImpl) SyncAndUpdatePresetContainerRegistry() { + presetSyncUrl := impl.presetDockerRegistryConfigBean.PresetRegistrySyncUrl + presetContainerRegistryByteArr, err := util2.ReadFromUrlWithRetry(presetSyncUrl) + centralDockerRegistryConfig, registryRepoName, registryExpiryTimeInSecs, err := impl.extractRegistryConfig(presetContainerRegistryByteArr) + if err != nil { + impl.logger.Errorw("err during unmarshal for preset container registry response from central-api", "err", err) + return + } + + impl.compareAndUpdateExpiryTime(registryExpiryTimeInSecs) + impl.compareAndUpdateRepoName(registryRepoName) + + registryId := util2.DockerPresetContainerRegistryId + dockerArtifactStore, err := impl.dockerRegistryConfig.FetchOneDockerAccount(registryId) + if err != nil { + impl.logger.Errorw("err in extracting docker registry from DB", "id", registryId, "err", err) + return + } + if changed := impl.compareCentralRegistryAndConfigured(centralDockerRegistryConfig, dockerArtifactStore); changed { + centralDockerRegistryConfig.Id = registryId + centralDockerRegistryConfig.User = 1 // system + _, err := impl.dockerRegistryConfig.Update(centralDockerRegistryConfig) + if err != nil { + impl.logger.Errorw("err in updating central-api docker registry into DB", "id", registryId, "err", err) + return + } + impl.logger.Info("docker preset container registry updated from central api") + } else { + impl.logger.Debug("docker preset container registry not updated as there is not diff, will check after sometime again!!") + } + +} + +func (impl *PresetContainerRegistryHandlerImpl) extractRegistryConfig(arr []byte) (*DockerArtifactStoreBean, string, int, error) { + + var result map[string]interface{} + err := json.Unmarshal(arr, &result) + if err != nil { + return nil, "", 0, err + } + statusCode := result["code"] + sCodeStr := statusCode.(float64) + if sCodeStr != 200 { + return nil, "", 0, errors.New("api failed with code" + fmt.Sprint(sCodeStr)) + } + responseBean := result["result"] + responseBeanMap := responseBean.(map[string]interface{}) + expiryTimeIsSecs := responseBeanMap["expiryTimeInSecs"] + registryDefaultRepoName := responseBeanMap["presetRepoName"] + response1, _ := json.Marshal(responseBeanMap) + centralDockerRegistryConfig := &DockerArtifactStoreBean{} + err = json.Unmarshal(response1, centralDockerRegistryConfig) + return centralDockerRegistryConfig, registryDefaultRepoName.(string), int(expiryTimeIsSecs.(float64)), err +} + +func (impl *PresetContainerRegistryHandlerImpl) compareCentralRegistryAndConfigured(centralDockerRegistry *DockerArtifactStoreBean, + dbDockerRegistry *DockerArtifactStoreBean) bool { + if centralDockerRegistry.PluginId != dbDockerRegistry.PluginId { + return true + } + if centralDockerRegistry.RegistryURL != dbDockerRegistry.RegistryURL { + return true + } + if centralDockerRegistry.RegistryType != dbDockerRegistry.RegistryType { + return true + } + if centralDockerRegistry.Username != dbDockerRegistry.Username { + return true + } + if centralDockerRegistry.Password != dbDockerRegistry.Password { + return true + } + if centralDockerRegistry.AWSRegion != dbDockerRegistry.AWSRegion { + return true + } + if centralDockerRegistry.AWSAccessKeyId != dbDockerRegistry.AWSAccessKeyId { + return true + } + if centralDockerRegistry.AWSSecretAccessKey != dbDockerRegistry.AWSSecretAccessKey { + return true + } + if centralDockerRegistry.Active != dbDockerRegistry.Active { + return true + } + if centralDockerRegistry.Cert != dbDockerRegistry.Cert { + return true + } + if centralDockerRegistry.Connection != dbDockerRegistry.Connection { + return true + } + + return false +} + +func (impl *PresetContainerRegistryHandlerImpl) compareAndUpdateExpiryTime(syncedExpiryTimeInSecs int) int { + cachedExpiryTimeInSecs := impl.presetDockerRegistryConfigBean.PresetRegistryImageExpiryTimeInSecs + + if syncedExpiryTimeInSecs != cachedExpiryTimeInSecs { + attributesDto, err := impl.attributeService.GetByKey(PresetRegistryExpiryTimeKey) + request := &attributes.AttributesDto{ + Id: attributesDto.Id, + Key: PresetRegistryExpiryTimeKey, + Value: strconv.Itoa(syncedExpiryTimeInSecs), + UserId: 1, + } + updateAttributeValue, err := impl.attributeService.UpdateAttributes(request) + if err != nil { + return syncedExpiryTimeInSecs + } + expiryTimeInSecStr := updateAttributeValue.Value + updatedExpiryTimeInSec, err := strconv.Atoi(expiryTimeInSecStr) + impl.presetDockerRegistryConfigBean.PresetRegistryImageExpiryTimeInSecs = updatedExpiryTimeInSec + } + return syncedExpiryTimeInSecs +} + +func (impl *PresetContainerRegistryHandlerImpl) compareAndUpdateRepoName(syncedRepoName string) string { + cachedRegistryRepoName := impl.presetDockerRegistryConfigBean.PresetRegistryRepoName + + if cachedRegistryRepoName != syncedRepoName { + attributesDto, err := impl.attributeService.GetByKey(PresetRegistryRepoNameKey) + request := &attributes.AttributesDto{ + Id: attributesDto.Id, + Key: PresetRegistryRepoNameKey, + Value: syncedRepoName, + UserId: 1, + } + updateAttributeValue, err := impl.attributeService.UpdateAttributes(request) + if err != nil { + return syncedRepoName + } + updatedRegistryRepoName := updateAttributeValue.Value + impl.presetDockerRegistryConfigBean.PresetRegistryRepoName = updatedRegistryRepoName + } + return syncedRepoName +} diff --git a/scripts/sql/63_docker_preset_container_registry.down.sql b/scripts/sql/63_docker_preset_container_registry.down.sql new file mode 100644 index 0000000000..368a6502ef --- /dev/null +++ b/scripts/sql/63_docker_preset_container_registry.down.sql @@ -0,0 +1,3 @@ +delete from docker_artifact_store where id = 'devtron-preset-container-registry'; + +delete from attributes where key IN ('PresetRegistryExpiryTime','PresetRegistryRepoName') \ No newline at end of file diff --git a/scripts/sql/63_docker_preset_container_registry.up.sql b/scripts/sql/63_docker_preset_container_registry.up.sql new file mode 100644 index 0000000000..43a537cde9 --- /dev/null +++ b/scripts/sql/63_docker_preset_container_registry.up.sql @@ -0,0 +1,6 @@ +Insert into docker_artifact_store(id, plugin_id, registry_url, registry_type, username, password, active, is_default, created_on, created_by, updated_on, updated_by, connection) values('devtron-preset-container-registry', 'cd.go.artifact.docker.registry', 'ttl.sh', 'other','a','a', 't', 'f', now(), 1, now(), 1, 'secure') + +Insert into attributes(key,value,active,created_on,created_by,updated_on) values('PresetRegistryRepoName','devtron-preset-registry-repo',true,now(),1,now()); + +Insert into attributes(key,value,active,created_on,created_by,updated_on) values('PresetRegistryExpiryTime','86400',true,now(),1,now()); + diff --git a/util/CommonConstant.go b/util/CommonConstant.go index c7f458f3ee..452e06461e 100644 --- a/util/CommonConstant.go +++ b/util/CommonConstant.go @@ -26,4 +26,5 @@ const ( ConfigMapSecretUsageTypeEnvironment string = "environment" ConfigMapSecretUsageTypeVolume string = "volume" YamlSeparator string = "---\n" + DockerPresetContainerRegistryId string = "devtron-preset-container-registry" ) diff --git a/wire_gen.go b/wire_gen.go index 9d3db85df7..6f8e0b9cc7 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -341,14 +341,15 @@ func InitializeApp() (*App, error) { appStoreApplicationVersionRepositoryImpl := appStoreDiscoverRepository.NewAppStoreApplicationVersionRepositoryImpl(sugaredLogger, db) environmentServiceImpl := cluster2.NewEnvironmentServiceImpl(environmentRepositoryImpl, clusterServiceImplExtended, sugaredLogger, k8sUtil, k8sInformerFactoryImpl, userAuthServiceImpl) helmAppServiceImpl := client3.NewHelmAppServiceImpl(sugaredLogger, clusterServiceImplExtended, helmAppClientImpl, pumpImpl, enforcerUtilHelmImpl, serverDataStoreServerDataStore, serverEnvConfigServerEnvConfig, appStoreApplicationVersionRepositoryImpl, environmentServiceImpl, pipelineRepositoryImpl) - pipelineBuilderImpl := pipeline.NewPipelineBuilderImpl(sugaredLogger, dbPipelineOrchestratorImpl, dockerArtifactStoreRepositoryImpl, materialRepositoryImpl, appRepositoryImpl, pipelineRepositoryImpl, propertiesConfigServiceImpl, ciTemplateRepositoryImpl, ciPipelineRepositoryImpl, serviceClientImpl, chartRepositoryImpl, ciArtifactRepositoryImpl, ecrConfig, envConfigOverrideRepositoryImpl, environmentRepositoryImpl, pipelineConfigRepositoryImpl, utilMergeUtil, appWorkflowRepositoryImpl, ciConfig, cdWorkflowRepositoryImpl, appServiceImpl, imageScanResultRepositoryImpl, argoK8sClientImpl, gitFactory, attributesServiceImpl, acdAuthConfig, gitOpsConfigRepositoryImpl, pipelineStrategyHistoryServiceImpl, prePostCiScriptHistoryServiceImpl, prePostCdScriptHistoryServiceImpl, deploymentTemplateHistoryServiceImpl, appLevelMetricsRepositoryImpl, pipelineStageServiceImpl, chartRefRepositoryImpl, chartTemplateServiceImpl, chartServiceImpl, helmAppServiceImpl, deploymentGroupRepositoryImpl) + dockerRegistryConfigImpl := pipeline.NewDockerRegistryConfigImpl(dockerArtifactStoreRepositoryImpl, sugaredLogger) + presetContainerRegistryHandlerImpl := pipeline.NewPresetContainerRegistryHandlerImpl(sugaredLogger, dockerRegistryConfigImpl, attributesServiceImpl) + pipelineBuilderImpl := pipeline.NewPipelineBuilderImpl(sugaredLogger, dbPipelineOrchestratorImpl, dockerArtifactStoreRepositoryImpl, materialRepositoryImpl, appRepositoryImpl, pipelineRepositoryImpl, propertiesConfigServiceImpl, ciTemplateRepositoryImpl, ciPipelineRepositoryImpl, serviceClientImpl, chartRepositoryImpl, ciArtifactRepositoryImpl, ecrConfig, envConfigOverrideRepositoryImpl, environmentRepositoryImpl, pipelineConfigRepositoryImpl, utilMergeUtil, appWorkflowRepositoryImpl, ciConfig, cdWorkflowRepositoryImpl, appServiceImpl, imageScanResultRepositoryImpl, argoK8sClientImpl, gitFactory, attributesServiceImpl, acdAuthConfig, gitOpsConfigRepositoryImpl, pipelineStrategyHistoryServiceImpl, prePostCiScriptHistoryServiceImpl, prePostCdScriptHistoryServiceImpl, deploymentTemplateHistoryServiceImpl, appLevelMetricsRepositoryImpl, pipelineStageServiceImpl, chartRefRepositoryImpl, chartTemplateServiceImpl, chartServiceImpl, helmAppServiceImpl, deploymentGroupRepositoryImpl, presetContainerRegistryHandlerImpl) dbMigrationServiceImpl := pipeline.NewDbMogrationService(sugaredLogger, dbMigrationConfigRepositoryImpl) workflowServiceImpl := pipeline.NewWorkflowServiceImpl(sugaredLogger, ciConfig) - ciServiceImpl := pipeline.NewCiServiceImpl(sugaredLogger, workflowServiceImpl, ciPipelineMaterialRepositoryImpl, ciWorkflowRepositoryImpl, ciConfig, eventRESTClientImpl, eventSimpleFactoryImpl, mergeUtil, ciPipelineRepositoryImpl, prePostCiScriptHistoryServiceImpl, pipelineStageServiceImpl, userServiceImpl) + ciServiceImpl := pipeline.NewCiServiceImpl(sugaredLogger, workflowServiceImpl, ciPipelineMaterialRepositoryImpl, ciWorkflowRepositoryImpl, ciConfig, eventRESTClientImpl, eventSimpleFactoryImpl, mergeUtil, ciPipelineRepositoryImpl, prePostCiScriptHistoryServiceImpl, pipelineStageServiceImpl, userServiceImpl, presetContainerRegistryHandlerImpl) ciLogServiceImpl := pipeline.NewCiLogServiceImpl(sugaredLogger, ciServiceImpl, ciConfig) ciHandlerImpl := pipeline.NewCiHandlerImpl(sugaredLogger, ciServiceImpl, ciPipelineMaterialRepositoryImpl, gitSensorClientImpl, ciWorkflowRepositoryImpl, workflowServiceImpl, ciLogServiceImpl, ciConfig, ciArtifactRepositoryImpl, userServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciPipelineRepositoryImpl, appListingRepositoryImpl) gitRegistryConfigImpl := pipeline.NewGitRegistryConfigImpl(sugaredLogger, gitProviderRepositoryImpl, gitSensorClientImpl) - dockerRegistryConfigImpl := pipeline.NewDockerRegistryConfigImpl(dockerArtifactStoreRepositoryImpl, sugaredLogger) cdHandlerImpl := pipeline.NewCdHandlerImpl(sugaredLogger, cdConfig, userServiceImpl, cdWorkflowRepositoryImpl, cdWorkflowServiceImpl, ciLogServiceImpl, ciArtifactRepositoryImpl, ciPipelineMaterialRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, ciWorkflowRepositoryImpl, ciConfig, helmAppServiceImpl, pipelineOverrideRepositoryImpl, workflowDagExecutorImpl) configMapServiceImpl := pipeline.NewConfigMapServiceImpl(chartRepositoryImpl, sugaredLogger, chartRepoRepositoryImpl, utilMergeUtil, pipelineConfigRepositoryImpl, configMapRepositoryImpl, envConfigOverrideRepositoryImpl, commonServiceImpl, appRepositoryImpl, configMapHistoryServiceImpl) appWorkflowServiceImpl := appWorkflow2.NewAppWorkflowServiceImpl(sugaredLogger, appWorkflowRepositoryImpl, dbPipelineOrchestratorImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl)