Config profile with repo API integration (#797)

Author: eranturgeman

.github/workflows/test.yml

@@ -139,7 +139,6 @@ jobs:
       # Generate mocks
       - name: Generate mocks
         run: go generate ./...
-        if: ${{ matrix.suite.name != 'Unit' }}
 
       - name: Run Tests
         if: ${{ matrix.suite.name != 'GitHub Integration' || matrix.os == 'ubuntu' }}

go.mod

@@ -9,15 +9,16 @@ require (
 	github.com/jfrog/build-info-go v1.10.7
 	github.com/jfrog/froggit-go v1.16.2
 	github.com/jfrog/gofrog v1.7.6
-	github.com/jfrog/jfrog-cli-core/v2 v2.57.0
-	github.com/jfrog/jfrog-cli-security v1.13.5
-	github.com/jfrog/jfrog-client-go v1.48.2
+	github.com/jfrog/jfrog-cli-core/v2 v2.57.2
+	github.com/jfrog/jfrog-cli-security v1.13.6
+	github.com/jfrog/jfrog-client-go v1.48.4
 	github.com/jordan-wright/email v4.0.1-0.20210109023952-943e75fe5223+incompatible
 	github.com/owenrumney/go-sarif/v2 v2.3.1
 	github.com/stretchr/testify v1.10.0
 	github.com/urfave/cli/v2 v2.27.4
 	github.com/xeipuuv/gojsonschema v1.2.0
 	golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f
+	gopkg.in/yaml.v2 v2.4.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 

go.sum

@@ -129,12 +129,12 @@ github.com/jfrog/gofrog v1.7.6 h1:QmfAiRzVyaI7JYGsB7cxfAJePAZTzFz0gRWZSE27c6s=
 github.com/jfrog/gofrog v1.7.6/go.mod h1:ntr1txqNOZtHplmaNd7rS4f8jpA5Apx8em70oYEe7+4=
 github.com/jfrog/jfrog-apps-config v1.0.1 h1:mtv6k7g8A8BVhlHGlSveapqf4mJfonwvXYLipdsOFMY=
 github.com/jfrog/jfrog-apps-config v1.0.1/go.mod h1:8AIIr1oY9JuH5dylz2S6f8Ym2MaadPLR6noCBO4C22w=
-github.com/jfrog/jfrog-cli-core/v2 v2.57.0 h1:3ON0J6Sjc2+4HZrzh4eSbdciXx3sJsJUIJ3TPQXh/5c=
-github.com/jfrog/jfrog-cli-core/v2 v2.57.0/go.mod h1:SThaC/fniC96oN8YgCsHjvOxp5rBM7IppuIybn1oxT0=
-github.com/jfrog/jfrog-cli-security v1.13.5 h1:CjWyCpURUtSCiu6xLqgvcwWFfy56hi+kFcMMTqEFhOo=
-github.com/jfrog/jfrog-cli-security v1.13.5/go.mod h1:gc14ZiaO0vXRrOWBoOlsPKrbRdKDgcl+jRFuCaAH0Vk=
-github.com/jfrog/jfrog-client-go v1.48.2 h1:YVAIiNWuBEa4NbWL54I+YzvXHaxoHDk532USDKTvgLU=
-github.com/jfrog/jfrog-client-go v1.48.2/go.mod h1:1a7bmQHkRmPEza9wva2+WVrYzrGbosrMymq57kyG5gU=
+github.com/jfrog/jfrog-cli-core/v2 v2.57.2 h1:2shy1CRWm/8yf6WWfVyAW3AdmryQiI73Tkhfb62vgPE=
+github.com/jfrog/jfrog-cli-core/v2 v2.57.2/go.mod h1:sgi0gw96J00Yzx2cKG5xTG/x9XD0YiJbglJOnXUeaD0=
+github.com/jfrog/jfrog-cli-security v1.13.6 h1:gm8TnGTlprMJbRga0cujeoN2xal7Pagd2kDkUfclvrw=
+github.com/jfrog/jfrog-cli-security v1.13.6/go.mod h1:NarJyhl8Kh0HL6br74oeStIosLmCjA7atLYxZIFHJc4=
+github.com/jfrog/jfrog-client-go v1.48.4 h1:uXvBr2ebFKpBRUhWgC9TSSJe32IbSYGlbDp9tDzBcaY=
+github.com/jfrog/jfrog-client-go v1.48.4/go.mod h1:2ySOMva54L3EYYIlCBYBTcTgqfrrQ19gtpA/MWfA/ec=
 github.com/jordan-wright/email v4.0.1-0.20210109023952-943e75fe5223+incompatible h1:jdpOPRN1zP63Td1hDQbZW73xKmzDvZHzVdNYxhnTMDA=
 github.com/jordan-wright/email v4.0.1-0.20210109023952-943e75fe5223+incompatible/go.mod h1:1c7szIrayyPPB/987hsnvNzLushdWf4o/79s3P08L8A=
 github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k=
@@ -413,6 +413,7 @@ gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME=
 gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

scanpullrequest/scanpullrequest.go

@@ -210,7 +210,7 @@ func auditTargetBranch(repoConfig *utils.Repository, scanDetails *utils.ScanDeta
 	// Download target branch (if needed)
 	cleanupTarget := func() error { return nil }
 	if !repoConfig.IncludeAllVulnerabilities {
-		if targetBranchWd, cleanupTarget, err = prepareTargetForScan(repoConfig.PullRequestDetails, scanDetails); err != nil {
+		if targetBranchWd, cleanupTarget, err = prepareTargetForScan(repoConfig.Git, scanDetails); err != nil {
 			return
 		}
 	}
@@ -232,8 +232,8 @@ func auditTargetBranch(repoConfig *utils.Repository, scanDetails *utils.ScanDeta
 	return
 }
 
-func prepareTargetForScan(pullRequestDetails vcsclient.PullRequestInfo, scanDetails *utils.ScanDetails) (targetBranchWd string, cleanupTarget func() error, err error) {
-	target := pullRequestDetails.Target
+func prepareTargetForScan(gitDetails utils.Git, scanDetails *utils.ScanDetails) (targetBranchWd string, cleanupTarget func() error, err error) {
+	target := gitDetails.PullRequestDetails.Target
 	// Download target branch
 	if targetBranchWd, cleanupTarget, err = utils.DownloadRepoToTempDir(scanDetails.Client(), target.Owner, target.Repository, target.Name); err != nil {
 		return
@@ -243,18 +243,23 @@ func prepareTargetForScan(pullRequestDetails vcsclient.PullRequestInfo, scanDeta
 	}
 	log.Debug("Using most common ancestor commit as target branch commit")
 	// Get common parent commit between source and target and use it (checkout) to the target branch commit
-	if e := tryCheckoutToMostCommonAncestor(scanDetails, pullRequestDetails.Source.Name, target.Name, targetBranchWd); e != nil {
-		log.Warn(fmt.Sprintf("Failed to get best common ancestor commit between source branch: %s and target branch: %s, defaulting to target branch commit. Error: %s", pullRequestDetails.Source.Name, target.Name, e.Error()))
+	if e := tryCheckoutToMostCommonAncestor(scanDetails, gitDetails.PullRequestDetails.Source.Name, target.Name, targetBranchWd, gitDetails.RepositoryCloneUrl); e != nil {
+		log.Warn(fmt.Sprintf("Failed to get best common ancestor commit between source branch: %s and target branch: %s, defaulting to target branch commit. Error: %s", gitDetails.PullRequestDetails.Source.Name, target.Name, e.Error()))
 	}
 	return
 }
 
-func tryCheckoutToMostCommonAncestor(scanDetails *utils.ScanDetails, baseBranch, headBranch, targetBranchWd string) (err error) {
-	repositoryInfo, err := scanDetails.Client().GetRepositoryInfo(context.Background(), scanDetails.RepoOwner, scanDetails.RepoName)
-	if err != nil {
-		return
+func tryCheckoutToMostCommonAncestor(scanDetails *utils.ScanDetails, baseBranch, headBranch, targetBranchWd, cloneRepoUrl string) (err error) {
+	if cloneRepoUrl != "" {
+		scanDetails.Git.RepositoryCloneUrl = cloneRepoUrl
+	} else {
+		var repositoryInfo vcsclient.RepositoryInfo
+		repositoryInfo, err = scanDetails.Client().GetRepositoryInfo(context.Background(), scanDetails.RepoOwner, scanDetails.RepoName)
+		if err != nil {
+			return
+		}
+		scanDetails.Git.RepositoryCloneUrl = repositoryInfo.CloneInfo.HTTP
 	}
-	scanDetails.Git.RepositoryCloneUrl = repositoryInfo.CloneInfo.HTTP
 	// Change working directory to the temp target branch directory
 	cwd, err := os.Getwd()
 	if err != nil {

scanrepository/scanrepository.go

@@ -127,6 +127,7 @@ func (cfp *ScanRepositoryCmd) setCommandPrerequisites(repository *utils.Reposito
 		SetXrayGraphScanParams(repository.Watches, repository.JFrogProjectKey, len(repository.AllowedLicenses) > 0).
 		SetFailOnInstallationErrors(*repository.FailOnSecurityIssues).
 		SetFixableOnly(repository.FixableOnly).
+		SetConfigProfile(repository.ConfigProfile).
 		SetSkipAutoInstall(repository.SkipAutoInstall).
 		SetAllowPartialResults(repository.AllowPartialResults).
 		SetDisableJas(repository.DisableJas)
@@ -137,11 +138,17 @@ func (cfp *ScanRepositoryCmd) setCommandPrerequisites(repository *utils.Reposito
 	if cfp.scanDetails, err = cfp.scanDetails.SetMinSeverity(repository.MinSeverity); err != nil {
 		return
 	}
-	repositoryInfo, err := client.GetRepositoryInfo(context.Background(), cfp.scanDetails.RepoOwner, cfp.scanDetails.RepoName)
-	if err != nil {
-		return
+	if repository.Git.RepositoryCloneUrl != "" {
+		cfp.scanDetails.Git.RepositoryCloneUrl = repository.Git.RepositoryCloneUrl
+	} else {
+		var repositoryInfo vcsclient.RepositoryInfo
+		repositoryInfo, err = client.GetRepositoryInfo(context.Background(), cfp.scanDetails.RepoOwner, cfp.scanDetails.RepoName)
+		if err != nil {
+			return
+		}
+		cfp.scanDetails.Git.RepositoryCloneUrl = repositoryInfo.CloneInfo.HTTP
 	}
-	cfp.scanDetails.Git.RepositoryCloneUrl = repositoryInfo.CloneInfo.HTTP
+
 	// Set the flag for aggregating fixes to generate a unified pull request for fixing vulnerabilities
 	cfp.aggregateFixes = repository.Git.AggregateFixes
 	// Set the outputwriter interface for the relevant vcs git provider

utils/consts.go

@@ -24,14 +24,15 @@ const (
 	azurePipelines ciProvider = "azure-pipelines"
 
 	// JFrog platform environment variables
-	JFrogUserEnv           = "JF_USER"
-	JFrogUrlEnv            = "JF_URL"
-	jfrogXrayUrlEnv        = "JF_XRAY_URL"
-	jfrogArtifactoryUrlEnv = "JF_ARTIFACTORY_URL"
-	jfrogReleasesRepoEnv   = "JF_RELEASES_REPO"
-	JFrogPasswordEnv       = "JF_PASSWORD"
-	JFrogTokenEnv          = "JF_ACCESS_TOKEN"
-	JfrogConfigProfileEnv  = "JF_CONFIG_PROFILE"
+	JFrogUserEnv             = "JF_USER"
+	JFrogUrlEnv              = "JF_URL"
+	jfrogXrayUrlEnv          = "JF_XRAY_URL"
+	jfrogArtifactoryUrlEnv   = "JF_ARTIFACTORY_URL"
+	jfrogReleasesRepoEnv     = "JF_RELEASES_REPO"
+	JFrogPasswordEnv         = "JF_PASSWORD"
+	JFrogTokenEnv            = "JF_ACCESS_TOKEN"
+	JfrogUseConfigProfileEnv = "JF_USE_CONFIG_PROFILE"
+	JfrogConfigProfileEnv    = "JF_CONFIG_PROFILE"
 
 	// Git environment variables
 	GitProvider                      = "JF_GIT_PROVIDER"

utils/params.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"gopkg.in/yaml.v2"
 	"net/http"
 	"net/url"
 	"os"
@@ -25,7 +26,6 @@ import (
 	"github.com/jfrog/froggit-go/vcsutils"
 	coreconfig "github.com/jfrog/jfrog-cli-core/v2/utils/config"
 	"github.com/jfrog/jfrog-client-go/utils/log"
-	"gopkg.in/yaml.v3"
 )
 
 const (
@@ -426,11 +426,6 @@ func GetFrogbotDetails(commandName string) (frogbotDetails *FrogbotDetails, err
 		return
 	}
 
-	configProfile, err := getConfigProfileIfExistsAndValid(xrayVersion, xscVersion, jfrogServer)
-	if err != nil {
-		return
-	}
-
 	gitParamsFromEnv, err := extractGitParamsFromEnvs(commandName)
 	if err != nil {
 		return
@@ -458,9 +453,16 @@ func GetFrogbotDetails(commandName string) (frogbotDetails *FrogbotDetails, err
 		return
 	}
 
-	// We apply the configProfile to all received repositories. This loop must be deleted when we will no longer accept multiple repositories in a single scan
+	configProfile, repoCloneUrl, err := getConfigProfileIfExistsAndValid(xrayVersion, xscVersion, jfrogServer, client, gitParamsFromEnv)
+	if err != nil {
+		return
+	}
+
+	// We apply the configProfile to all received repositories. If no config profile was fetched, a nil value is passed
+	// TODO This loop must be deleted when we will no longer accept multiple repositories in a single scan
 	for i := range configAggregator {
 		configAggregator[i].Scan.ConfigProfile = configProfile
+		configAggregator[i].Git.RepositoryCloneUrl = repoCloneUrl
 	}
 
 	frogbotDetails = &FrogbotDetails{XrayVersion: xrayVersion, XscVersion: xscVersion, Repositories: configAggregator, GitClient: client, ServerDetails: jfrogServer, ReleasesRepo: os.Getenv(jfrogReleasesRepoEnv)}
@@ -791,28 +793,55 @@ func readConfigFromTarget(client vcsclient.VcsClient, gitParamsFromEnv *Git) (co
 	return
 }
 
-// This function fetches a config profile if JF_CONFIG_PROFILE is provided.
-// If so - it verifies there is only a single module with a '.' path from root. If these conditions doesn't hold we return an error.
-func getConfigProfileIfExistsAndValid(xrayVersion, xscVersion string, jfrogServer *coreconfig.ServerDetails) (configProfile *services.ConfigProfile, err error) {
+// This function attempts to fetch a config profile if JF_USE_CONFIG_PROFILE is set to true.
+// If we need to use a profile, we first try to get the profile by name that can be provided through JF_CONFIG_PROFILE. If name is provided but profile doesn't exist we return an error.
+// If we need to use a profile, but name is not provided, we check if there is a config profile associated to the repo URL.
+// When a profile is found we verify several conditions on it.
+// If a profile was requested but not found by url nor by name we return an error.
+func getConfigProfileIfExistsAndValid(xrayVersion, xscVersion string, jfrogServer *coreconfig.ServerDetails, gitClient vcsclient.VcsClient, gitParams *Git) (configProfile *services.ConfigProfile, repoCloneUrl string, err error) {
+	var useConfigProfile bool
+	if useConfigProfile, err = getBoolEnv(JfrogUseConfigProfileEnv, false); err != nil || !useConfigProfile {
+		log.Debug(fmt.Sprintf("Configuration Profile usage is disabled. All configurations will be derived from environment variables and files.\nTo enable a Configuration Profile, please set %s to TRUE", JfrogUseConfigProfileEnv))
+		return
+	}
+
+	// Attempt to get the config profile by profile's name
 	profileName := getTrimmedEnv(JfrogConfigProfileEnv)
-	if profileName == "" {
-		log.Debug(fmt.Sprintf("No %s environment variable was provided. All configurations will be induced from Env vars and files", JfrogConfigProfileEnv))
+	if profileName != "" {
+		log.Debug(fmt.Sprintf("Configuration profile was requested. Searching profile by provided name '%s'", profileName))
+		if configProfile, err = xsc.GetConfigProfileByName(xrayVersion, xscVersion, jfrogServer, profileName); err != nil || configProfile == nil {
+			return
+		}
+		err = verifyConfigProfileValidity(configProfile)
 		return
 	}
 
-	if configProfile, err = xsc.GetConfigProfile(xrayVersion, xscVersion, jfrogServer, profileName); err != nil {
+	// Getting repository's url in order to get repository HTTP url
+	repositoryInfo, err := gitClient.GetRepositoryInfo(context.Background(), gitParams.RepoOwner, gitParams.RepoName)
+	if err != nil {
+		return nil, "", err
+	}
+	repoCloneUrl = repositoryInfo.CloneInfo.HTTP
+
+	// Attempt to get a config profile associated with the repo URL
+	log.Debug(fmt.Sprintf("Configuration profile was requested. Searching profile associated to repository '%s'", jfrogServer.Url))
+	if configProfile, err = xsc.GetConfigProfileByUrl(xrayVersion, jfrogServer, repoCloneUrl); err != nil || configProfile == nil {
 		return
 	}
+	err = verifyConfigProfileValidity(configProfile)
+	return
+}
 
+func verifyConfigProfileValidity(configProfile *services.ConfigProfile) (err error) {
 	// Currently, only a single Module that represents the entire project is supported
 	if len(configProfile.Modules) != 1 {
 		err = fmt.Errorf("more than one module was found '%s' profile. Frogbot currently supports only one module per config profile", configProfile.ProfileName)
 		return
 	}
 	if configProfile.Modules[0].PathFromRoot != "." {
-		err = fmt.Errorf("module '%s' in profile '%s' contains the following path from root: '%s'. Frogbot currently supports only a single module with a '.' path from root", configProfile.Modules[0].ModuleName, profileName, configProfile.Modules[0].PathFromRoot)
+		err = fmt.Errorf("module '%s' in profile '%s' contains the following path from root: '%s'. Frogbot currently supports only a single module with a '.' path from root", configProfile.Modules[0].ModuleName, configProfile.ProfileName, configProfile.Modules[0].PathFromRoot)
 		return
 	}
-	log.Info(fmt.Sprintf("Using Config profile '%s'. jfrog-apps-config will be ignored if exists", profileName))
+	log.Info(fmt.Sprintf("Using Config profile '%s'. jfrog-apps-config will be ignored if exists", configProfile.ProfileName))
 	return
 }