Skip to content

Commit 7ed4b3d

Browse files
vikramdevtronvikramdevtron
authored
Merge pull request #102 from devtron-labs/release-candidate-v0.26.0
sync: Release candidate v0.26.0
2 parents f06c39c + 7239e03 commit 7ed4b3d

File tree

47 files changed

+602
-260
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

47 files changed

+602
-260
lines changed

‎chart-sync/go.mod

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ go 1.22.4
55
toolchain go1.22.6
66

77
replace (
8-
github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20241219031102-d5eb208ee11b
8+
github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20241230042545-446c0258ec13
99
helm.sh/helm/v3 v3.14.3 => github.com/devtron-labs/helm/v3 v3.14.1-0.20240401080259-90238cf69e42
1010
)
1111

‎chart-sync/go.sum

+2-2
Original file line numberDiff line numberDiff line change
@@ -54,8 +54,8 @@ github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr
5454
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
5555
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
5656
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
57-
github.com/devtron-labs/devtron-services/common-lib v0.0.0-20241219031102-d5eb208ee11b h1:0Mua8RfGFNDbaAprezc6NM5TnQdNbqo+qMVtbERx6Yg=
58-
github.com/devtron-labs/devtron-services/common-lib v0.0.0-20241219031102-d5eb208ee11b/go.mod h1:NJSMdv+zTUK3p7rML12RZSeAUKHeLaoY3sR/oK0xhwo=
57+
github.com/devtron-labs/devtron-services/common-lib v0.0.0-20241230042545-446c0258ec13 h1:0nYnqC8SuDbXJY9vfC6Wg4xMgsmCi2s+d57SpoRfJ84=
58+
github.com/devtron-labs/devtron-services/common-lib v0.0.0-20241230042545-446c0258ec13/go.mod h1:NJSMdv+zTUK3p7rML12RZSeAUKHeLaoY3sR/oK0xhwo=
5959
github.com/devtron-labs/helm/v3 v3.14.1-0.20240401080259-90238cf69e42 h1:pJmK44QaSztOiZe0iQHNf0sdy5KwkAeceydyhOG4RaY=
6060
github.com/devtron-labs/helm/v3 v3.14.1-0.20240401080259-90238cf69e42/go.mod h1:v6myVbyseSBJTzhmeE39UcPLNv6cQK6qss3dvgAySaE=
6161
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=

‎chart-sync/vendor/modules.txt

+2-2
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ github.com/containerd/platforms
9393
# github.com/davecgh/go-spew v1.1.1
9494
## explicit
9595
github.com/davecgh/go-spew/spew
96-
# github.com/devtron-labs/common-lib v0.0.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20241219031102-d5eb208ee11b
96+
# github.com/devtron-labs/common-lib v0.0.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20241230042545-446c0258ec13
9797
## explicit; go 1.21
9898
github.com/devtron-labs/common-lib/helmLib/registry
9999
github.com/devtron-labs/common-lib/utils/http
@@ -785,4 +785,4 @@ sigs.k8s.io/structured-merge-diff/v4/value
785785
# sigs.k8s.io/yaml v1.3.0
786786
## explicit; go 1.12
787787
sigs.k8s.io/yaml
788-
# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20241219031102-d5eb208ee11b
788+
# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20241230042545-446c0258ec13

‎ci-runner/executor/adaptor/ImageScanAdaptor.go

+44
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
/*
2+
* Copyright (c) 2024. Devtron Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package adaptor
18+
19+
import (
20+
"github.com/devtron-labs/ci-runner/helper"
21+
"github.com/devtron-labs/common-lib/constants"
22+
"github.com/devtron-labs/common-lib/imageScan/bean"
23+
)
24+
25+
func GetImageScanEvent(dest, digest string, commonWorkflowRequest *helper.CommonWorkflowRequest) *helper.ScanEvent {
26+
if commonWorkflowRequest == nil {
27+
return &helper.ScanEvent{}
28+
}
29+
return &helper.ScanEvent{
30+
ImageScanEvent: bean.ImageScanEvent{
31+
Image: dest,
32+
ImageDigest: digest,
33+
PipelineId: commonWorkflowRequest.PipelineId,
34+
UserId: commonWorkflowRequest.TriggeredBy,
35+
DockerRegistryId: commonWorkflowRequest.DockerRegistryId,
36+
DockerConnection: commonWorkflowRequest.DockerConnection,
37+
DockerCert: commonWorkflowRequest.DockerCert,
38+
SourceType: constants.SourceTypeImage,
39+
SourceSubType: constants.SourceSubTypeCi,
40+
},
41+
ImageScanMaxRetries: commonWorkflowRequest.ImageScanMaxRetries,
42+
ImageScanRetryDelay: commonWorkflowRequest.ImageScanRetryDelay,
43+
}
44+
}

‎ci-runner/executor/stage/bean/bean.go

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
/*
2+
* Copyright (c) 2024. Devtron Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package bean
18+
19+
const (
20+
ExternalCiArtifact = "externalCiArtifact"
21+
ImageDigest = "imageDigest"
22+
UseAppDockerConfig = "useAppDockerConfig"
23+
CiProjectDetails = "ciProjectDetails"
24+
)

‎ci-runner/executor/stage/ciStages.go

+73-56
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,9 @@ import (
2222
"errors"
2323
"fmt"
2424
"github.com/devtron-labs/ci-runner/executor"
25+
adaptor2 "github.com/devtron-labs/ci-runner/executor/adaptor"
2526
cicxt "github.com/devtron-labs/ci-runner/executor/context"
27+
bean2 "github.com/devtron-labs/ci-runner/executor/stage/bean"
2628
util2 "github.com/devtron-labs/ci-runner/executor/util"
2729
"github.com/devtron-labs/ci-runner/helper"
2830
"github.com/devtron-labs/ci-runner/helper/adaptor"
@@ -144,6 +146,7 @@ func (impl *CiStage) HandleCIEvent(ciCdRequest *helper.CiCdTriggerEvent, exitCod
144146
return
145147
}
146148

149+
// TODO: take as tech debt and break this function into parts for better code readability
147150
func (impl *CiStage) runCIStages(ciContext cicxt.CiContext, ciCdRequest *helper.CiCdTriggerEvent) (artifactUploaded bool, err error) {
148151

149152
metrics := &helper.CIMetrics{}
@@ -281,6 +284,18 @@ func (impl *CiStage) runCIStages(ciContext cicxt.CiContext, ciCdRequest *helper.
281284
if err != nil {
282285
return artifactUploaded, err
283286
}
287+
if scriptEnvs.RuntimeEnv[bean2.ExternalCiArtifact] != "" {
288+
runtimeImage, runtimeDigest, err := impl.handleRuntimeParametersForCiJob(scriptEnvs.RuntimeEnv, ciCdRequest)
289+
if err != nil {
290+
log.Println(util.DEVTRON, "error in handling runtime parameters for ci job and getting runtime image and digest")
291+
return artifactUploaded, err
292+
}
293+
if len(runtimeImage) > 0 {
294+
dest = runtimeImage
295+
digest = runtimeDigest
296+
}
297+
}
298+
284299
// scan only if ci scan enabled
285300
if helper.IsEventTypeEligibleToScanImage(ciCdRequest.Type) &&
286301
ciCdRequest.CommonWorkflowRequest.ScanEnabled {
@@ -292,51 +307,7 @@ func (impl *CiStage) runCIStages(ciContext cicxt.CiContext, ciCdRequest *helper.
292307

293308
log.Println(util.DEVTRON, " event")
294309
metrics.TotalDuration = time.Since(metrics.TotalStartTime).Seconds()
295-
// When externalCiArtifact is provided (run time Env at time of build) then this image will be used further in the pipeline
296-
// imageDigest and ciProjectDetails are optional fields
297-
if scriptEnvs.RuntimeEnv["externalCiArtifact"] != "" {
298-
log.Println(util.DEVTRON, "external ci artifact found! exiting now with success event")
299-
dest = scriptEnvs.RuntimeEnv["externalCiArtifact"]
300-
digest = scriptEnvs.RuntimeEnv["imageDigest"]
301-
if len(digest) == 0 {
302-
var useAppDockerConfigForPrivateRegistries bool
303-
var err error
304-
useAppDockerConfig, ok := ciCdRequest.CommonWorkflowRequest.RuntimeEnvironmentVariables["useAppDockerConfig"]
305-
if ok && len(useAppDockerConfig) > 0 {
306-
useAppDockerConfigForPrivateRegistries, err = strconv.ParseBool(useAppDockerConfig)
307-
if err != nil {
308-
fmt.Println(fmt.Sprintf("Error in parsing useAppDockerConfig runtime param to bool from string useAppDockerConfigForPrivateRegistries:- %s, err:", useAppDockerConfig), err)
309-
}
310-
}
311-
var dockerAuthConfig *bean.DockerAuthConfig
312-
if useAppDockerConfigForPrivateRegistries {
313-
dockerAuthConfig = impl.dockerHelper.GetDockerAuthConfigForPrivateRegistries(ciCdRequest.CommonWorkflowRequest)
314-
}
315-
startTime := time.Now()
316-
//user has not provided imageDigest in that case fetch from docker.
317-
imgDigest, err := impl.dockerHelper.ExtractDigestFromImage(dest, ciCdRequest.CommonWorkflowRequest.UseDockerApiToGetDigest, dockerAuthConfig)
318-
if err != nil {
319-
fmt.Println(fmt.Sprintf("Error in extracting digest from image %s, err:", dest), err)
320-
return artifactUploaded, err
321-
}
322-
log.Println(fmt.Sprintf("time since extract digest from image process:- %s", time.Since(startTime).String()))
323-
digest = imgDigest
324-
}
325-
var tempDetails []*helper.CiProjectDetailsMin
326-
err := json.Unmarshal([]byte(scriptEnvs.RuntimeEnv["ciProjectDetails"]), &tempDetails)
327-
if err != nil {
328-
fmt.Println("Error unmarshalling ciProjectDetails JSON:", err)
329-
fmt.Println("ignoring the error and continuing without saving ciProjectDetails")
330-
}
331310

332-
if len(tempDetails) > 0 && len(ciCdRequest.CommonWorkflowRequest.CiProjectDetails) > 0 {
333-
detail := tempDetails[0]
334-
ciCdRequest.CommonWorkflowRequest.CiProjectDetails[0].CommitHash = detail.CommitHash
335-
ciCdRequest.CommonWorkflowRequest.CiProjectDetails[0].Message = detail.Message
336-
ciCdRequest.CommonWorkflowRequest.CiProjectDetails[0].Author = detail.Author
337-
ciCdRequest.CommonWorkflowRequest.CiProjectDetails[0].CommitTime = detail.CommitTime
338-
}
339-
}
340311
event := adaptor.NewCiCompleteEvent(ciCdRequest.CommonWorkflowRequest).WithMetrics(*metrics).
341312
WithDockerImage(dest).WithDigest(digest).WithIsArtifactUploaded(artifactUploaded).
342313
WithImageDetailsFromCR(resultsFromPlugin).WithPluginArtifacts(pluginArtifacts)
@@ -467,18 +438,8 @@ func (impl *CiStage) runPostCiSteps(ciCdRequest *helper.CiCdTriggerEvent, script
467438
func runImageScanning(dest string, digest string, ciCdRequest *helper.CiCdTriggerEvent, metrics *helper.CIMetrics, artifactUploaded bool) error {
468439
imageScanningStage := func() error {
469440
log.Println("Image Scanning Started for digest", digest)
470-
scanEvent := &helper.ScanEvent{
471-
Image: dest,
472-
ImageDigest: digest,
473-
PipelineId: ciCdRequest.CommonWorkflowRequest.PipelineId,
474-
UserId: ciCdRequest.CommonWorkflowRequest.TriggeredBy,
475-
DockerRegistryId: ciCdRequest.CommonWorkflowRequest.DockerRegistryId,
476-
DockerConnection: ciCdRequest.CommonWorkflowRequest.DockerConnection,
477-
DockerCert: ciCdRequest.CommonWorkflowRequest.DockerCert,
478-
ImageScanMaxRetries: ciCdRequest.CommonWorkflowRequest.ImageScanMaxRetries,
479-
ImageScanRetryDelay: ciCdRequest.CommonWorkflowRequest.ImageScanRetryDelay,
480-
}
481-
err := helper.SendEventToClairUtility(scanEvent)
441+
scanEvent := adaptor2.GetImageScanEvent(dest, digest, ciCdRequest.CommonWorkflowRequest)
442+
err := helper.ExecuteImageScanningViaRest(scanEvent)
482443
if err != nil {
483444
log.Println("error in running Image Scan", "err", err)
484445
return helper.NewCiStageError(err).
@@ -639,3 +600,59 @@ func (impl *CiStage) AddExtraEnvVariableFromRuntimeParamsToCiCdEvent(ciRequest *
639600
}
640601
return ciRequest.RuntimeEnvironmentVariables, nil
641602
}
603+
604+
// When externalCiArtifact is provided (run time Env at time of build) then this image will be used further in the pipeline
605+
// imageDigest and ciProjectDetails are optional fields
606+
func (impl *CiStage) handleRuntimeParametersForCiJob(runtimeEnv map[string]string, ciCdRequest *helper.CiCdTriggerEvent) (string, string, error) {
607+
log.Println(util.DEVTRON, "external ci artifact found! exiting now with success event")
608+
dest := runtimeEnv[bean2.ExternalCiArtifact]
609+
digest := runtimeEnv[bean2.ImageDigest]
610+
var err error
611+
if len(digest) == 0 {
612+
digest, err = impl.extractDigestForCiJob(ciCdRequest.CommonWorkflowRequest, dest)
613+
if err != nil {
614+
log.Println(util.DEVTRON, " extract digest for ci job error", "dest", dest, "err", err)
615+
return dest, digest, err
616+
}
617+
}
618+
var tempDetails []*helper.CiProjectDetailsMin
619+
err = json.Unmarshal([]byte(runtimeEnv[bean2.CiProjectDetails]), &tempDetails)
620+
if err != nil {
621+
fmt.Println("Error unmarshalling ciProjectDetails JSON:", err)
622+
fmt.Println("ignoring the error and continuing without saving ciProjectDetails")
623+
}
624+
if len(tempDetails) > 0 && len(ciCdRequest.CommonWorkflowRequest.CiProjectDetails) > 0 {
625+
detail := tempDetails[0]
626+
ciCdRequest.CommonWorkflowRequest.CiProjectDetails[0].CommitHash = detail.CommitHash
627+
ciCdRequest.CommonWorkflowRequest.CiProjectDetails[0].Message = detail.Message
628+
ciCdRequest.CommonWorkflowRequest.CiProjectDetails[0].Author = detail.Author
629+
ciCdRequest.CommonWorkflowRequest.CiProjectDetails[0].CommitTime = detail.CommitTime
630+
}
631+
return dest, digest, nil
632+
}
633+
634+
func (impl *CiStage) extractDigestForCiJob(workflowRequest *helper.CommonWorkflowRequest, image string) (string, error) {
635+
var useAppDockerConfigForPrivateRegistries bool
636+
var err error
637+
useAppDockerConfig, ok := workflowRequest.RuntimeEnvironmentVariables[bean2.UseAppDockerConfig]
638+
if ok && len(useAppDockerConfig) > 0 {
639+
useAppDockerConfigForPrivateRegistries, err = strconv.ParseBool(useAppDockerConfig)
640+
if err != nil {
641+
fmt.Println(fmt.Sprintf("Error in parsing useAppDockerConfig runtime param to bool from string useAppDockerConfigForPrivateRegistries:- %s, err:", useAppDockerConfig), err)
642+
// would use default val of useAppDockerConfigForPrivateRegistries i.e false in case error arises
643+
}
644+
}
645+
var dockerAuthConfig *bean.DockerAuthConfig
646+
if useAppDockerConfigForPrivateRegistries {
647+
dockerAuthConfig = impl.dockerHelper.GetDockerAuthConfigForPrivateRegistries(workflowRequest)
648+
}
649+
startTime := time.Now()
650+
//user has not provided imageDigest in that case fetch from docker.
651+
imgDigest, err := impl.dockerHelper.ExtractDigestFromImage(image, workflowRequest.UseDockerApiToGetDigest, dockerAuthConfig)
652+
if err != nil {
653+
fmt.Println(fmt.Sprintf("Error in extracting digest from image %s, err:", image), err)
654+
return "", err
655+
}
656+
log.Println(fmt.Sprintf("time since extract digest from image process:- %s", time.Since(startTime).String()))
657+
return imgDigest, nil
658+
}

‎ci-runner/go.mod

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ go 1.21
44

55
toolchain go1.21.8
66

7-
replace github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20241219031102-d5eb208ee11b
7+
replace github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20241230042545-446c0258ec13
88

99
require (
1010
github.com/Knetic/govaluate v3.0.0+incompatible

‎ci-runner/go.sum

+2-2
Original file line numberDiff line numberDiff line change
@@ -57,8 +57,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
5757
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
5858
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
5959
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
60-
github.com/devtron-labs/devtron-services/common-lib v0.0.0-20241219031102-d5eb208ee11b h1:0Mua8RfGFNDbaAprezc6NM5TnQdNbqo+qMVtbERx6Yg=
61-
github.com/devtron-labs/devtron-services/common-lib v0.0.0-20241219031102-d5eb208ee11b/go.mod h1:NJSMdv+zTUK3p7rML12RZSeAUKHeLaoY3sR/oK0xhwo=
60+
github.com/devtron-labs/devtron-services/common-lib v0.0.0-20241230042545-446c0258ec13 h1:0nYnqC8SuDbXJY9vfC6Wg4xMgsmCi2s+d57SpoRfJ84=
61+
github.com/devtron-labs/devtron-services/common-lib v0.0.0-20241230042545-446c0258ec13/go.mod h1:NJSMdv+zTUK3p7rML12RZSeAUKHeLaoY3sR/oK0xhwo=
6262
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
6363
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
6464
github.com/docker/cli v24.0.6+incompatible h1:fF+XCQCgJjjQNIMjzaSmiKJSCcfcXb3TWTcc7GAneOY=

‎ci-runner/helper/EventHelper.go

+5-17
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"crypto/tls"
2121
"encoding/json"
2222
"fmt"
23+
bean2 "github.com/devtron-labs/common-lib/imageScan/bean"
2324
"github.com/devtron-labs/common-lib/utils/remoteConnection/bean"
2425
"log"
2526
"net/http"
@@ -597,7 +598,7 @@ func PublishEventsOnRest(jsonBody []byte, topic string, cdRequest *ExtEnvRequest
597598
return nil
598599
}
599600

600-
func SendEventToClairUtility(event *ScanEvent) error {
601+
func ExecuteImageScanningViaRest(event *ScanEvent) error {
601602
jsonBody, err := json.Marshal(event)
602603
if err != nil {
603604
log.Println(util.DEVTRON, "err", err)
@@ -642,22 +643,9 @@ func SendEventToClairUtility(event *ScanEvent) error {
642643
}
643644

644645
type ScanEvent struct {
645-
Image string `json:"image"`
646-
ImageDigest string `json:"imageDigest"`
647-
AppId int `json:"appId"`
648-
EnvId int `json:"envId"`
649-
PipelineId int `json:"pipelineId"`
650-
CiArtifactId int `json:"ciArtifactId"`
651-
UserId int `json:"userId"`
652-
AccessKey string `json:"accessKey"`
653-
SecretKey string `json:"secretKey"`
654-
Token string `json:"token"`
655-
AwsRegion string `json:"awsRegion"`
656-
DockerRegistryId string `json:"dockerRegistryId"`
657-
DockerConnection string `json:"dockerConnection"`
658-
DockerCert string `json:"dockerCert"`
659-
ImageScanMaxRetries int `json:"imageScanMaxRetries,omitempty"`
660-
ImageScanRetryDelay int `json:"imageScanRetryDelay,omitempty"`
646+
bean2.ImageScanEvent
647+
ImageScanMaxRetries int `json:"imageScanMaxRetries,omitempty"`
648+
ImageScanRetryDelay int `json:"imageScanRetryDelay,omitempty"`
661649
}
662650

663651
func (dockerBuildConfig *DockerBuildConfig) GetProvenanceFlag() string {

‎ci-runner/vendor/github.com/devtron-labs/common-lib/constants/constants.go

+16
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)