Run contextual analysis and secret detection in Docker scans (#10)

Author: guyshe-jfrog

commands/audit/audit.go

@@ -3,16 +3,22 @@ package audit
 import (
 	"errors"
 	"fmt"
+	"os"
+
 	"github.com/jfrog/jfrog-cli-core/v2/utils/coreutils"
 	"github.com/jfrog/jfrog-cli-security/scangraph"
-	"github.com/jfrog/jfrog-cli-security/utils"
+
+	"github.com/jfrog/jfrog-cli-security/jas"
+
+	"github.com/jfrog/jfrog-cli-security/jas/applicability"
+	"github.com/jfrog/jfrog-cli-security/jas/runner"
+	"github.com/jfrog/jfrog-cli-security/jas/secrets"
 	clientutils "github.com/jfrog/jfrog-client-go/utils"
 	"github.com/jfrog/jfrog-client-go/utils/log"
 	"github.com/jfrog/jfrog-client-go/xray"
 	"github.com/jfrog/jfrog-client-go/xray/services"
 	xscservices "github.com/jfrog/jfrog-client-go/xsc/services"
 	"golang.org/x/sync/errgroup"
-	"os"
 
 	xrayutils "github.com/jfrog/jfrog-cli-security/utils"
 )
@@ -182,15 +188,15 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error)
 		return
 	}
 	results.XrayVersion = auditParams.xrayVersion
-	results.ExtendedScanResults.EntitledForJas, err = isEntitledForJas(xrayManager, auditParams.xrayVersion)
+	results.ExtendedScanResults.EntitledForJas, err = jas.IsEntitledForJas(xrayManager, auditParams.xrayVersion)
 	if err != nil {
 		return
 	}
 
 	errGroup := new(errgroup.Group)
 	if results.ExtendedScanResults.EntitledForJas {
 		// Download (if needed) the analyzer manager in a background routine.
-		errGroup.Go(utils.DownloadAnalyzerManagerIfNeeded)
+		errGroup.Go(xrayutils.DownloadAnalyzerManagerIfNeeded)
 	}
 
 	results.MultiScanId = auditParams.XrayGraphScanParams().MultiScanId
@@ -205,16 +211,7 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error)
 
 	// Run scanners only if the user is entitled for Advanced Security
 	if results.ExtendedScanResults.EntitledForJas {
-		results.JasError = runJasScannersAndSetResults(results, auditParams.DirectDependencies(), serverDetails, auditParams.workingDirs, auditParams.Progress(), auditParams.thirdPartyApplicabilityScan, auditParams.XrayGraphScanParams().MultiScanId)
-	}
-	return
-}
-
-func isEntitledForJas(xrayManager *xray.XrayServicesManager, xrayVersion string) (entitled bool, err error) {
-	if e := clientutils.ValidateMinimumVersion(clientutils.Xray, xrayVersion, xrayutils.EntitlementsMinVersion); e != nil {
-		log.Debug(e)
-		return
+		results.JasError = runner.RunJasScannersAndSetResults(results.ExtendedScanResults, results.GetScaScannedTechnologies(), results.GetScaScansXrayResults(), auditParams.DirectDependencies(), serverDetails, auditParams.workingDirs, auditParams.Progress(), auditParams.thirdPartyApplicabilityScan, auditParams.XrayGraphScanParams().MultiScanId, applicability.ApplicabilityScannerType, secrets.SecretsScannerType)
 	}
-	entitled, err = xrayManager.IsEntitled(xrayutils.ApplicabilityFeatureId)
 	return
 }

commands/audit/jasrunner.go

@@ -3,16 +3,17 @@ package scan
 import (
 	"bytes"
 	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
 	"github.com/jfrog/jfrog-cli-core/v2/common/spec"
 	xrayutils "github.com/jfrog/jfrog-cli-security/utils"
 	clientutils "github.com/jfrog/jfrog-client-go/utils"
 	"github.com/jfrog/jfrog-client-go/utils/errorutils"
 	"github.com/jfrog/jfrog-client-go/utils/io/fileutils"
 	"github.com/jfrog/jfrog-client-go/utils/log"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"strings"
 )
 
 const (
@@ -81,6 +82,7 @@ func (dsc *DockerScanCommand) Run() (err error) {
 		Pattern(imageTarPath).
 		Target(dsc.targetRepoPath).
 		BuildSpec()).SetThreads(1)
+	dsc.ScanCommand.SetRunJasScans(true)
 	err = dsc.setCredentialEnvsForIndexerApp()
 	if err != nil {
 		return errorutils.CheckError(err)

commands/scan/dockerscan.go

@@ -10,12 +10,19 @@ import (
 	"regexp"
 	"strings"
 
+	"golang.org/x/exp/maps"
+	"golang.org/x/exp/slices"
+
+	"github.com/jfrog/jfrog-cli-security/jas"
+	"github.com/jfrog/jfrog-cli-security/jas/applicability"
+	"github.com/jfrog/jfrog-cli-security/jas/runner"
+	"github.com/jfrog/jfrog-cli-security/jas/secrets"
 	"github.com/jfrog/jfrog-cli-security/scangraph"
 	xrayUtils "github.com/jfrog/jfrog-client-go/xray/services/utils"
+	"golang.org/x/sync/errgroup"
 
 	"github.com/jfrog/gofrog/parallel"
 	"github.com/jfrog/jfrog-cli-core/v2/common/format"
-	outputFormat "github.com/jfrog/jfrog-cli-core/v2/common/format"
 	"github.com/jfrog/jfrog-cli-core/v2/common/spec"
 	"github.com/jfrog/jfrog-cli-core/v2/utils/config"
 	"github.com/jfrog/jfrog-cli-core/v2/utils/coreutils"
@@ -35,8 +42,9 @@ type FileContext func(string) parallel.TaskFunc
 type indexFileHandlerFunc func(file string)
 
 type ScanInfo struct {
-	Target string
-	Result *services.ScanResponse
+	Target              string
+	Result              *services.ScanResponse
+	ExtendedScanResults *utils.ExtendedScanResults
 }
 
 const (
@@ -52,7 +60,7 @@ type ScanCommand struct {
 	// The location of the downloaded Xray indexer binary on the local file system.
 	indexerPath            string
 	indexerTempDir         string
-	outputFormat           outputFormat.OutputFormat
+	outputFormat           format.OutputFormat
 	projectKey             string
 	minSeverityFilter      string
 	watches                []string
@@ -63,6 +71,7 @@ type ScanCommand struct {
 	bypassArchiveLimits    bool
 	fixableOnly            bool
 	progress               ioUtils.ProgressMgr
+	commandSupportsJAS     bool
 }
 
 func (scanCmd *ScanCommand) SetMinSeverityFilter(minSeverityFilter string) *ScanCommand {
@@ -75,6 +84,11 @@ func (scanCmd *ScanCommand) SetFixableOnly(fixable bool) *ScanCommand {
 	return scanCmd
 }
 
+func (scanCmd *ScanCommand) SetRunJasScans(run bool) *ScanCommand {
+	scanCmd.commandSupportsJAS = run
+	return scanCmd
+}
+
 func (scanCmd *ScanCommand) SetProgress(progress ioUtils.ProgressMgr) {
 	scanCmd.progress = progress
 }
@@ -84,7 +98,7 @@ func (scanCmd *ScanCommand) SetThreads(threads int) *ScanCommand {
 	return scanCmd
 }
 
-func (scanCmd *ScanCommand) SetOutputFormat(format outputFormat.OutputFormat) *ScanCommand {
+func (scanCmd *ScanCommand) SetOutputFormat(format format.OutputFormat) *ScanCommand {
 	scanCmd.outputFormat = format
 	return scanCmd
 }
@@ -182,6 +196,16 @@ func (scanCmd *ScanCommand) Run() (err error) {
 		return err
 	}
 
+	scanResults := xrutils.NewAuditResults()
+	scanResults.XrayVersion = xrayVersion
+
+	scanResults.ExtendedScanResults.EntitledForJas, err = jas.IsEntitledForJas(xrayManager, xrayVersion)
+	errGroup := new(errgroup.Group)
+	if scanResults.ExtendedScanResults.EntitledForJas {
+		// Download (if needed) the analyzer manager in a background routine.
+		errGroup.Go(xrutils.DownloadAnalyzerManagerIfNeeded)
+	}
+
 	// Validate Xray minimum version for graph scan command
 	err = clientutils.ValidateMinimumVersion(clientutils.Xray, xrayVersion, scangraph.GraphScanMinXrayVersion)
 	if err != nil {
@@ -226,16 +250,20 @@ func (scanCmd *ScanCommand) Run() (err error) {
 	fileCollectingErrorsQueue := clientutils.NewErrorsQueue(1)
 	// Start walking on the filesystem to "produce" files that match the given pattern
 	// while the consumer uses the indexer to index those files.
-	scanCmd.prepareScanTasks(fileProducerConsumer, indexedFileProducerConsumer, resultsArr, fileProducerErrors, indexedFileProducerErrors, fileCollectingErrorsQueue, xrayVersion)
+	scanCmd.prepareScanTasks(fileProducerConsumer, indexedFileProducerConsumer, scanResults.ExtendedScanResults.EntitledForJas, resultsArr, fileProducerErrors, indexedFileProducerErrors, fileCollectingErrorsQueue, xrayVersion)
 	scanCmd.performScanTasks(fileProducerConsumer, indexedFileProducerConsumer)
 
 	// Handle results
 	flatResults := []xrutils.ScaScanResult{}
+
 	for _, arr := range resultsArr {
 		for _, res := range arr {
 			flatResults = append(flatResults, xrutils.ScaScanResult{Target: res.Target, XrayResults: []services.ScanResponse{*res.Result}})
+			scanResults.ExtendedScanResults.ApplicabilityScanResults = append(scanResults.ExtendedScanResults.ApplicabilityScanResults, res.ExtendedScanResults.ApplicabilityScanResults...)
+			scanResults.ExtendedScanResults.SecretsScanResults = append(scanResults.ExtendedScanResults.SecretsScanResults, res.ExtendedScanResults.SecretsScanResults...)
 		}
 	}
+
 	if scanCmd.progress != nil {
 		if err = scanCmd.progress.Quit(); err != nil {
 			return err
@@ -251,10 +279,13 @@ func (scanCmd *ScanCommand) Run() (err error) {
 	scanErrors = appendErrorSlice(scanErrors, fileProducerErrors)
 	scanErrors = appendErrorSlice(scanErrors, indexedFileProducerErrors)
 
-	scanResults := xrutils.NewAuditResults()
-	scanResults.XrayVersion = xrayVersion
 	scanResults.ScaResults = flatResults
 
+	// Wait for the Download of the AnalyzerManager to complete.
+	if err = errGroup.Wait(); err != nil {
+		err = errors.New("failed while trying to get Analyzer Manager: " + err.Error())
+	}
+
 	if err = xrutils.NewResultsWriter(scanResults).
 		SetOutputFormat(scanCmd.outputFormat).
 		SetIncludeVulnerabilities(scanCmd.includeVulnerabilities).
@@ -296,14 +327,14 @@ func (scanCmd *ScanCommand) CommandName() string {
 	return "xr_scan"
 }
 
-func (scanCmd *ScanCommand) prepareScanTasks(fileProducer, indexedFileProducer parallel.Runner, resultsArr [][]*ScanInfo, fileErrors, indexedFileErrors [][]formats.SimpleJsonError, fileCollectingErrorsQueue *clientutils.ErrorsQueue, xrayVersion string) {
+func (scanCmd *ScanCommand) prepareScanTasks(fileProducer, indexedFileProducer parallel.Runner, entitledForJas bool, resultsArr [][]*ScanInfo, fileErrors, indexedFileErrors [][]formats.SimpleJsonError, fileCollectingErrorsQueue *clientutils.ErrorsQueue, xrayVersion string) {
 	go func() {
 		defer fileProducer.Done()
 		// Iterate over file-spec groups and produce indexing tasks.
 		// When encountering an error, log and move to next group.
 		specFiles := scanCmd.spec.Files
 		for i := range specFiles {
-			artifactHandlerFunc := scanCmd.createIndexerHandlerFunc(&specFiles[i], indexedFileProducer, resultsArr, fileErrors, indexedFileErrors, xrayVersion)
+			artifactHandlerFunc := scanCmd.createIndexerHandlerFunc(&specFiles[i], entitledForJas, indexedFileProducer, resultsArr, fileErrors, indexedFileErrors, xrayVersion)
 			taskHandler := getAddTaskToProducerFunc(fileProducer, artifactHandlerFunc)
 
 			err := collectFilesForIndexing(specFiles[i], taskHandler)
@@ -315,7 +346,7 @@ func (scanCmd *ScanCommand) prepareScanTasks(fileProducer, indexedFileProducer p
 	}()
 }
 
-func (scanCmd *ScanCommand) createIndexerHandlerFunc(file *spec.File, indexedFileProducer parallel.Runner, resultsArr [][]*ScanInfo, fileErrors, indexedFileErrors [][]formats.SimpleJsonError, xrayVersion string) FileContext {
+func (scanCmd *ScanCommand) createIndexerHandlerFunc(file *spec.File, entitledForJas bool, indexedFileProducer parallel.Runner, resultsArr [][]*ScanInfo, fileErrors, indexedFileErrors [][]formats.SimpleJsonError, xrayVersion string) FileContext {
 	return func(filePath string) parallel.TaskFunc {
 		return func(threadId int) (err error) {
 			logMsgPrefix := clientutils.GetLogMsgPrefix(threadId, false)
@@ -360,12 +391,26 @@ func (scanCmd *ScanCommand) createIndexerHandlerFunc(file *spec.File, indexedFil
 					return err
 				}
 				scanResults, err := scangraph.RunScanGraphAndGetResults(scanGraphParams, xrayManager)
+
 				if err != nil {
 					log.Error(fmt.Sprintf("scanning '%s' failed with error: %s", graph.Id, err.Error()))
 					indexedFileErrors[threadId] = append(indexedFileErrors[threadId], formats.SimpleJsonError{FilePath: filePath, ErrorMessage: err.Error()})
 					return
 				}
-				resultsArr[threadId] = append(resultsArr[threadId], &ScanInfo{Target: filePath, Result: scanResults})
+
+				extendedScanResults := utils.ExtendedScanResults{}
+				if entitledForJas && scanCmd.commandSupportsJAS {
+                    // Run Jas scans
+					workingDirs := []string{filePath}
+					err = runner.RunJasScannersAndSetResults(&extendedScanResults, []coreutils.Technology{coreutils.Technology(scanResults.ScannedPackageType)}, []services.ScanResponse{*scanResults}, depsListFromVulnerabilities(*scanResults), scanCmd.serverDetails, workingDirs, nil, false, "", applicability.ApplicabilityDockerScanScanType, secrets.SecretsScannerDockerScanType)
+
+					if err != nil {
+						log.Error(fmt.Sprintf("scanning '%s' failed with error: %s", graph.Id, err.Error()))
+						indexedFileErrors[threadId] = append(indexedFileErrors[threadId], formats.SimpleJsonError{FilePath: filePath, ErrorMessage: err.Error()})
+						return
+					}
+				}
+				resultsArr[threadId] = append(resultsArr[threadId], &ScanInfo{Target: filePath, Result: scanResults, ExtendedScanResults: &extendedScanResults})
 				return
 			}
 
@@ -469,6 +514,20 @@ func appendErrorSlice(scanErrors []formats.SimpleJsonError, errorsToAdd [][]form
 	return scanErrors
 }
 
+func depsListFromVulnerabilities(scanResult ...services.ScanResponse) (depsList []string) {
+	for _, result := range scanResult {
+		for _, vulnerability := range result.Vulnerabilities {
+			dependencies := maps.Keys(vulnerability.Components)
+			for _, dependency := range dependencies {
+				if !slices.Contains(depsList, dependency) {
+					depsList = append(depsList, dependency)
+				}
+			}
+		}
+	}
+	return
+}
+
 func ConditionalUploadDefaultScanFunc(serverDetails *config.ServerDetails, fileSpec *spec.SpecFiles, threads int, scanOutputFormat format.OutputFormat) error {
 	return NewScanCommand().SetServerDetails(serverDetails).SetSpec(fileSpec).SetThreads(threads).SetOutputFormat(scanOutputFormat).SetFail(true).SetPrintExtendedTable(false).Run()
 }

commands/scan/scan.go

@@ -28,6 +28,7 @@ func ConvertToVulnerabilityScanTableRow(rows []VulnerabilityOrViolationRow) (tab
 		tableRows = append(tableRows, vulnerabilityScanTableRow{
 			severity:               rows[i].Severity,
 			severityNumValue:       rows[i].SeverityNumValue,
+			applicable:             rows[i].Applicable,
 			impactedPackageName:    rows[i].ImpactedDependencyName,
 			impactedPackageVersion: rows[i].ImpactedDependencyVersion,
 			ImpactedPackageType:    rows[i].ImpactedDependencyType,

formats/conversion.go

@@ -20,7 +20,8 @@ type vulnerabilityTableRow struct {
 }
 
 type vulnerabilityScanTableRow struct {
-	severity string `col-name:"Severity"`
+	severity   string `col-name:"Severity"`
+	applicable string `col-name:"Contextual\nAnalysis" omitempty:"true"`
 	// For sorting
 	severityNumValue       int
 	directPackages         []directPackagesTableRow `embed-table:"true"`