v1.0.X-performance

Hardened (#159) - performance, sync, path filtering and whitelisting improvements

Author: interceptd

cmd/api.go

@@ -168,7 +168,7 @@ func processWithRegex(policy Policy, data []byte, rgPath string) error {
 }
 func executeAssureForAPI(policy Policy, rgPath, filePath string) (bool, error) {
 	// Create a temporary file to store the search patterns
-	searchPatternFile, err := createSearchPatternFile(policy.Regex)
+	searchPatternFile, err := createSearchPatternFile(policy.Regex, NormalizeFilename(policy.ID))
 	if err != nil {
 		return false, fmt.Errorf("error creating search pattern file: %w", err)
 	}

cmd/assure.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"sync"
 )
 
 // ProcessAssureType handles the assurance process for policies of type "assure"
@@ -23,7 +24,7 @@ func ProcessAssureType(policy Policy, rgPath string, targetDir string, filePaths
 
 func executeAssure(policy Policy, rgPath string, targetDir string, filesToAssure []string) error {
 	// Create a temporary file to store the search patterns
-	searchPatternFile, err := createSearchPatternFile(policy.Regex)
+	searchPatternFile, err := createSearchPatternFile(policy.Regex, NormalizeFilename(policy.ID))
 	if err != nil {
 		log.Error().Err(err).Msg("error creating search pattern file")
 		return fmt.Errorf("error creating search pattern file: %w", err)
@@ -49,6 +50,8 @@ func executeAssure(policy Policy, rgPath string, targetDir string, filesToAssure
 	writer := bufio.NewWriter(jsonoutfile)
 	defer writer.Flush()
 
+	processedIgnorePatterns := processIgnorePatterns(policyData.Config.Flags.Ignore)
+
 	codePatternAssureJSON := []string{
 		"--pcre2",
 		"--no-heading",
@@ -64,39 +67,24 @@ func executeAssure(policy Policy, rgPath string, targetDir string, filesToAssure
 		return fmt.Errorf("no target directory defined")
 	}
 
-	// Append the file targets
-	if len(filesToAssure) > 0 {
-		codePatternAssureJSON = append(codePatternAssureJSON, filesToAssure...)
-	} else if policy.FilePattern == "" {
-		codePatternAssureJSON = append(codePatternAssureJSON, targetDir)
+	codePatternAssureJSON = append(codePatternAssureJSON, processedIgnorePatterns...)
+
+	matchesFound := true
+
+	// Parallel execution for large file sets
+	if policy.FilePattern == "" {
+		log.Warn().Str("policy", policy.ID).Msg("ASSURE Policy without a filepattern is suboptimal")
+	}
+	if len(filesToAssure) > 25 {
+		matchesFound, err = executeParallelAssure(rgPath, codePatternAssureJSON, filesToAssure, writer)
 	} else {
-		return fmt.Errorf("no files matched policy pattern")
+		matchesFound, err = executeSingleAssure(rgPath, codePatternAssureJSON, filesToAssure, targetDir, policy, writer)
 	}
 
-	// Execute the ripgrep command for JSON output
-	cmdJSON := exec.Command(rgPath, codePatternAssureJSON...)
-	cmdJSON.Stdout = writer
-	cmdJSON.Stderr = os.Stderr
+	if err != nil {
 
-	log.Debug().Msgf("Creating JSON output for assure policy %s... ", policy.ID)
-	err = cmdJSON.Run()
+		log.Error().Err(err).Msg("error executing ripgrep batch")
 
-	// Check if ripgrep found any matches
-	matchesFound := true
-	if err != nil {
-		if exitError, ok := err.(*exec.ExitError); ok {
-			// Exit code 1 in ripgrep means "no matches found"
-			if exitError.ExitCode() == 1 {
-				matchesFound = false
-				err = nil // Reset error as this is the expected outcome for assure
-			} else {
-				log.Error().Err(err).Msg("error executing ripgrep for JSON output")
-				return fmt.Errorf("error executing ripgrep for JSON output: %w", err)
-			}
-		} else {
-			log.Error().Err(err).Msg("error executing ripgrep for JSON output")
-			return fmt.Errorf("error executing ripgrep for JSON output: %w", err)
-		}
 	}
 
 	// Patch the JSON output file
@@ -107,6 +95,7 @@ func executeAssure(policy Policy, rgPath string, targetDir string, filesToAssure
 	}
 
 	log.Debug().Msgf("JSON output for assure policy %s written to: %s ", policy.ID, jsonOutputFile)
+	log.Debug().Msgf("Scanned ~%d files for policy %s", len(filesToAssure), policy.ID)
 
 	// Determine the status based on whether matches were found
 	status := "NOT FOUND"
@@ -143,3 +132,115 @@ func executeAssure(policy Policy, rgPath string, targetDir string, filesToAssure
 
 	return nil
 }
+
+func executeParallelAssure(rgPath string, baseArgs []string, filesToScan []string, writer *bufio.Writer) (bool, error) {
+
+	const batchSize = 25
+	matched := true
+	var wg sync.WaitGroup
+	errChan := make(chan error, len(filesToScan)/batchSize+1)
+	var mu sync.Mutex
+
+	for i := 0; i < len(filesToScan); i += batchSize {
+		end := i + batchSize
+		if end > len(filesToScan) {
+			end = len(filesToScan)
+		}
+		batch := filesToScan[i:end]
+
+		// log.Debug().Msgf("RGM: %v", batch)
+
+		wg.Add(1)
+		go func(batch []string) {
+			defer wg.Done()
+			args := append(baseArgs, batch...)
+			cmd := exec.Command(rgPath, args...)
+			output, err := cmd.Output()
+
+			if err != nil {
+
+				if exitError, ok := err.(*exec.ExitError); ok {
+					// Exit code 1 in ripgrep means "no matches found"
+					if exitError.ExitCode() == 1 {
+						matched = false
+						err = nil // Reset error as this is the expected outcome for assure
+					}
+				}
+
+				if exitError, ok := err.(*exec.ExitError); ok && exitError.ExitCode() != 1 {
+					errChan <- fmt.Errorf("error executing ripgrep: %w", err)
+					return
+				}
+			}
+
+			mu.Lock()
+			_, writeErr := writer.Write(output)
+			if writeErr == nil {
+				writeErr = writer.Flush()
+			}
+			mu.Unlock()
+
+			if writeErr != nil {
+				errChan <- fmt.Errorf("error writing output: %w", writeErr)
+			}
+		}(batch)
+	}
+
+	wg.Wait()
+	close(errChan)
+
+	for err := range errChan {
+		if err != nil {
+			return matched, err
+		}
+	}
+
+	return matched, nil
+}
+
+func executeSingleAssure(rgPath string, baseArgs []string, filesToScan []string, targetDir string, policy Policy, writer *bufio.Writer) (bool, error) {
+
+	if len(filesToScan) > 0 {
+		baseArgs = append(baseArgs, filesToScan...)
+	} else {
+		log.Error().Str("policy", policy.ID).Msgf("no files matched policy pattern on target : %s", targetDir)
+	}
+
+	matched := true
+
+	// log.Debug().Msgf("RGS: %v", baseArgs)
+
+	cmdJSON := exec.Command(rgPath, baseArgs...)
+	cmdJSON.Stdout = writer
+	cmdJSON.Stderr = os.Stderr
+
+	log.Debug().Msgf("Creating JSON output for policy %s... ", policy.ID)
+	err := cmdJSON.Run()
+	if err != nil {
+		if exitError, ok := err.(*exec.ExitError); ok {
+
+			if exitError.ExitCode() == 1 {
+				matched = false
+				err = nil // Reset error as this is the expected outcome for assure
+			}
+
+			if exitError.ExitCode() == 2 {
+				log.Warn().Msgf("RG exited with code 2")
+				log.Debug().Msgf("RG Error Args: %v", baseArgs)
+				if len(exitError.Stderr) > 0 {
+					log.Debug().Msgf("RG exited with code 2 stderr: %s", string(exitError.Stderr))
+				}
+			}
+			if exitError.ExitCode() != 1 {
+				log.Error().Err(err).Msg("error executing ripgrep for JSON output")
+				return matched, fmt.Errorf("error executing ripgrep for JSON output: %w", err)
+			}
+
+		} else {
+			log.Error().Err(err).Msg("error executing ripgrep for JSON output")
+			return matched, fmt.Errorf("error executing ripgrep for JSON output: %w", err)
+		}
+	}
+
+	return matched, nil
+}

cmd/audit.go

@@ -209,9 +209,11 @@ func processPolicy(policy Policy, allFileInfos []FileInfo, rgPath string) {
 
 	if policy.Type == "json" || policy.Type == "yaml" || policy.Type == "ini" || policy.Type == "scan" || policy.Type == "assure" {
 
-		log.Debug().Msgf(" Processing files for policy %s: ", policy.ID)
-		for _, file := range filesToProcess {
-			log.Debug().Msgf("  %s: %s ", file.Path, file.Hash)
+		log.Debug().Str("policy", policy.ID).Msgf(" Processing files for policy %s ", policy.ID)
+		if len(filesToProcess) < 15 {
+			for _, file := range filesToProcess {
+				log.Debug().Str("policy", policy.ID).Msgf("  %s: %s ", file.Path, file.Hash)
+			}
 		}
 
 		normalizedID := NormalizeFilename(policy.ID)

cmd/aux.go

@@ -1,6 +1,7 @@
 package cmd
 
 import (
+	"bufio"
 	"crypto/sha256"
 	"encoding/hex"
 	"encoding/json"
@@ -22,29 +23,40 @@ import (
 	"gopkg.in/yaml.v3"
 )
 
-// patchJSONOutputFile reads the ripgrep JSON output, patches it to create valid JSON, and writes it back to the file
 func patchJSONOutputFile(filePath string) error {
-	// Read the contents of the file
-	content, err := os.ReadFile(filePath)
+	// Open the file
+	file, err := os.Open(filePath)
 	if err != nil {
-		return fmt.Errorf("error reading JSON file: %v", err)
+		return fmt.Errorf("error opening JSON file: %v", err)
 	}
+	defer file.Close()
 
-	// Split the input into separate JSON objects
-	objects := strings.Split(string(content), "}\n{")
+	var validObjects []map[string]interface{}
 
-	// Add the missing brackets to create a JSON array
-	jsonArray := "[" + strings.Join(objects, "},\n{") + "]"
+	// Read the file line by line
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		if line == "" {
+			continue
+		}
 
-	// Parse the JSON array to validate it
-	var parsed []interface{}
-	err = json.Unmarshal([]byte(jsonArray), &parsed)
-	if err != nil {
-		return fmt.Errorf("error parsing JSON: %v", err)
+		// Try to parse each line as a separate JSON object
+		var obj map[string]interface{}
+		if err := json.Unmarshal([]byte(line), &obj); err == nil {
+			validObjects = append(validObjects, obj)
+		} else {
+			// Log the error and skip this line
+			fmt.Printf("Skipping invalid JSON line: %s\n", line)
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		return fmt.Errorf("error reading file: %v", err)
 	}
 
-	// Re-marshal the parsed data to get a properly formatted JSON string
-	validJSON, err := json.MarshalIndent(parsed, "", "  ")
+	// Marshal the array of objects into a properly formatted JSON string
+	validJSON, err := json.MarshalIndent(validObjects, "", "  ")
 	if err != nil {
 		return fmt.Errorf("error marshaling JSON: %v", err)
 	}
@@ -334,7 +346,7 @@ func createOutputDirectories(isObserve bool) error {
 	for _, dir := range dirs {
 		if outputDir != "" {
 			dir = filepath.Join(outputDir, dir)
-			log.Debug().Msgf("Creating directory: %s", dir)
+			// log.Debug().Msgf("Creating directory: %s", dir)
 
 		}
 		if err := os.MkdirAll(dir, 0755); err != nil {
@@ -350,7 +362,7 @@ func cleanupOutputDirectories() error {
 	if outputDir != "" {
 		for i, dir := range dirsToClean {
 			dirsToClean[i] = filepath.Join(outputDir, dir)
-			log.Debug().Msgf("Cleaning up directories: %v", dirsToClean)
+			// log.Debug().Msgf("Cleaning up directories: %v", dirsToClean)
 		}
 	}
 	var wg sync.WaitGroup
@@ -367,7 +379,7 @@ func cleanupOutputDirectories() error {
 				return
 			}
 
-			log.Debug().Msgf("Cleaned up directory: %s ", d)
+			// log.Debug().Msgf("Cleaned up directory: %s ", d)
 		}(dir)
 	}