diff --git a/.golangci.yml b/.golangci.yml index 7fe1a41..6b04723 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -272,7 +272,6 @@ linters: - loggercheck # checks key value pairs for common logger libraries (kitlog,klog,logr,zap) - makezero # finds slice declarations with non-zero initial length - mirror # reports wrong mirror patterns of bytes/strings usage - - mnd # detects magic numbers - musttag # enforces field tags in (un)marshaled structs - nakedret # finds naked returns in functions greater than a specified function length - nestif # reports deeply nested if statements diff --git a/cli/commands.go b/cli/commands.go index 658ff37..458cbf1 100644 --- a/cli/commands.go +++ b/cli/commands.go @@ -180,8 +180,8 @@ func createTrackAutoCmd() *cobra.Command { cmd.Flags().StringVar(&benchmarkName, benchNameFlag, "", "Name of the benchmark") cmd.Flags().StringVar(&profileType, profileTypeFlag, "", "Profile type (cpu, memory, mutex, block)") cmd.Flags().StringVar(&outputFormat, outputFormatFlag, "detailed", `Output format: "summary" or "detailed"`) - cmd.Flags().BoolVar(&failOnRegression, failFlag, false, "Exit with non-zero code if regression exceeds threshold") - cmd.Flags().Float64Var(®ressionThreshold, thresholdFlag, 0.0, "Fail when worst flat regression exceeds this percent (e.g., 5.0)") + cmd.Flags().BoolVar(&failOnRegression, failFlag, false, "Exit with non-zero code if regression exceeds threshold (optional when using CI/CD config)") + cmd.Flags().Float64Var(®ressionThreshold, thresholdFlag, 0.0, "Fail when worst flat regression exceeds this percent (optional when using CI/CD config)") _ = cmd.MarkFlagRequired(baseTagFlag) _ = cmd.MarkFlagRequired(currentTagFlag) @@ -219,8 +219,8 @@ func createTrackManualCmd() *cobra.Command { cmd.Flags().StringVar(&Baseline, baseTagFlag, "", "Name of the baseline tag") cmd.Flags().StringVar(&Current, currentTagFlag, "", "Name of the current tag") cmd.Flags().StringVar(&outputFormat, outputFormatFlag, "", "Output format choice choice") - cmd.Flags().BoolVar(&failOnRegression, failFlag, false, "Exit with non-zero code if regression exceeds threshold") - cmd.Flags().Float64Var(®ressionThreshold, thresholdFlag, 0.0, "Fail when worst flat regression exceeds this percent (e.g., 5.0)") + cmd.Flags().BoolVar(&failOnRegression, failFlag, false, "Exit with non-zero code if regression exceeds threshold (optional when using CI/CD config)") + cmd.Flags().Float64Var(®ressionThreshold, thresholdFlag, 0.0, "Fail when worst flat regression exceeds this percent (optional when using CI/CD config)") _ = cmd.MarkFlagRequired(baseTagFlag) _ = cmd.MarkFlagRequired(currentTagFlag) diff --git a/docs/cicd_configuration.md b/docs/cicd_configuration.md new file mode 100644 index 0000000..3f85cf0 --- /dev/null +++ b/docs/cicd_configuration.md @@ -0,0 +1,403 @@ +# CI/CD Configuration Guide + +This document explains how to configure Prof for CI/CD environments to reduce noise and make performance regression detection more reliable. + +## Overview + +Prof's CI/CD configuration allows you to: + +- **Filter out noisy functions** that shouldn't cause CI/CD failures +- **Set different thresholds** for different benchmarks +- **Override command-line settings** with configuration files +- **Fail on unexpected improvements** if needed + +## Configuration Structure + +The CI/CD configuration is added to your existing `config_template.json` file under the `ci_config` section: + +```json +{ + "function_collection_filter": { + // ... existing function filtering ... + }, + "ci_config": { + "global": { + // Global CI/CD settings + }, + "benchmarks": { + "BenchmarkName": { + // Benchmark-specific CI/CD settings + } + } + } +} +``` + +## Global Configuration + +Global settings apply to all benchmarks unless overridden by benchmark-specific settings: + +```json +"global": { + "ignore_functions": [ + "runtime.gcBgMarkWorker", + "runtime.systemstack", + "testing.(*B).ResetTimer" + ], + "ignore_prefixes": [ + "runtime.", + "reflect.", + "testing." + ], + "min_change_threshold": 5.0, + "max_regression_threshold": 20.0, + "fail_on_improvement": false, +} +``` + +### Global Settings Explained + +| Setting | Description | Default | +| -------------------------- | ------------------------------------------------ | ------- | +| `ignore_functions` | Functions to ignore during CI/CD (exact matches) | `[]` | +| `ignore_prefixes` | Function prefixes to ignore (e.g., "runtime.") | `[]` | +| `min_change_threshold` | Minimum change % to trigger CI/CD failure | `0.0` | +| `max_regression_threshold` | Maximum acceptable regression % | `∞` | +| `fail_on_improvement` | Whether to fail on performance improvements | `false` | + +## Benchmark-Specific Configuration + +You can override global settings for specific benchmarks: + +```json +"benchmarks": { + "BenchmarkMyFunction": { + "ignore_functions": ["BenchmarkMyFunction"], + "min_change_threshold": 3.0, + "max_regression_threshold": 10.0, + "fail_on_improvement": true, + } +} +``` + +## Function Filtering + +### Ignoring Specific Functions + +Functions can be ignored by exact name: + +```json +"ignore_functions": [ + "runtime.gcBgMarkWorker", + "testing.(*B).ResetTimer", + "myproject.BenchmarkFunction" +] +``` + +### Ignoring Function Prefixes + +Functions can be ignored by package prefix: + +```json +"ignore_prefixes": [ + "runtime.", + "reflect.", + "testing.", + "syscall.", + "internal/cpu." +] +``` + +This will ignore all functions from the `runtime`, `reflect`, `testing`, `syscall`, and `internal/cpu` packages. + +## Threshold Configuration + +### Minimum Change Threshold + +Only functions with changes ≥ this threshold will cause CI/CD failures: + +```json +"min_change_threshold": 5.0 +``` + +This prevents CI/CD from failing on minor fluctuations (e.g., 1-2% changes). + +### Maximum Regression Threshold + +This overrides command-line `--regression-threshold` settings: + +```json +"max_regression_threshold": 15.0 +``` + +If a function regresses by 15% or more, CI/CD will fail regardless of command-line settings. + +### Command-Line Override Priority + +1. Benchmark-specific `max_regression_threshold` +2. Global `max_regression_threshold` +3. Command-line `--regression-threshold` + +The most restrictive (lowest) threshold wins. + +## Failing on Improvements + +Sometimes you want to detect unexpected performance improvements: + +```json +"fail_on_improvement": true +``` + +This is useful when: + +- Performance improvements might indicate bugs +- You want to track all significant changes +- You're debugging unexpected behavior + +## Complete Example + +Here's a complete configuration example: + +```json +{ + "function_collection_filter": { + "*": { + "include_prefixes": ["github.com/myorg/myproject"], + "ignore_functions": ["init", "TestMain"] + } + }, + "ci_config": { + "global": { + "ignore_functions": ["runtime.gcBgMarkWorker", "testing.(*B).ResetTimer"], + "ignore_prefixes": ["runtime.", "reflect.", "testing."], + "min_change_threshold": 5.0, + "max_regression_threshold": 20.0, + "fail_on_improvement": false + }, + "benchmarks": { + "BenchmarkCriticalPath": { + "min_change_threshold": 1.0, + "max_regression_threshold": 5.0 + } + } + } +} +``` + +## CI/CD Integration + +### GitHub Actions Example + +```yaml +name: Performance Regression Check +on: [pull_request] + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: ">=1.24" + + - name: Install prof + run: go install github.com/AlexsanderHamir/prof/cmd/prof@latest + + - name: Collect baseline + run: | + git fetch origin main --depth=1 + git checkout -qf origin/main + prof auto --benchmarks "BenchmarkMyFunction" --profiles "cpu" --count 5 --tag baseline + + - name: Collect current + run: | + git checkout - + prof auto --benchmarks "BenchmarkMyFunction" --profiles "cpu" --count 5 --tag PR + + - name: Check for regressions + run: | + prof track auto --base baseline --current PR \ + --profile-type cpu --bench-name "BenchmarkMyFunction" \ + --output-format summary +``` + +### Configuration File Location + +The configuration file must be at your project root (same directory as `go.mod`): + +``` +your-project/ +├── go.mod +├── config_template.json # ← CI/CD config goes here +├── cmd/ +├── internal/ +└── ... +``` + +## Complete Working Example + +Here's a complete example that shows how to set up CI/CD performance tracking without requiring CLI flags: + +### 1. Configuration File (`config_template.json`) + +```json +{ + "ci_config": { + "global": { + "ignore_prefixes": ["runtime.", "reflect.", "testing."], + "min_change_threshold": 5.0, + "max_regression_threshold": 15.0, + "fail_on_improvement": false + }, + "benchmarks": { + "BenchmarkMyFunction": { + "min_change_threshold": 3.0, + "max_regression_threshold": 10.0, + "ignore_functions": ["setup", "teardown"] + } + } + } +} +``` + +### 2. CI/CD Pipeline (`.github/workflows/performance.yml`) + +```yaml +- name: Check for regressions + run: | + prof track auto --base baseline --current PR \ + --profile-type cpu --bench-name "BenchmarkMyFunction" \ + --output-format summary +``` + +Notice that no `--fail-on-regression` or `--regression-threshold` flags are needed. The tool will automatically use the thresholds from your configuration file. + +## Best Practices + +### 1. Start with Global Configuration + +Begin with global settings that apply to all benchmarks: + +```json +"global": { + "ignore_prefixes": ["runtime.", "reflect.", "testing."], + "min_change_threshold": 5.0 +} +``` + +### 2. CLI Flags vs Configuration + +When using CI/CD configuration, the `--fail-on-regression` and `--regression-threshold` flags become optional: + +**With CLI flags (overrides config):** + +```bash +prof track auto --base baseline --current PR \ + --profile-type cpu --bench-name "BenchmarkMyFunction" \ + --output-format summary --fail-on-regression --regression-threshold 5.0 +``` + +**Without CLI flags (uses config only):** + +```bash +prof track auto --base baseline --current PR \ + --profile-type cpu --bench-name "BenchmarkMyFunction" \ + --output-format summary +``` + +The second approach will use the thresholds defined in your `config_template.json` file. This makes CI/CD pipelines cleaner and more maintainable. + +### 3. Add Benchmark-Specific Overrides + +Only override global settings when necessary: + +```json +"benchmarks": { + "BenchmarkCriticalPath": { + "min_change_threshold": 1.0 // More sensitive for critical paths + } +} +``` + +### 4. Use Function Filtering Sparingly + +Don't ignore too many functions - you might miss real regressions: + +```json +"ignore_functions": [ + "runtime.gcBgMarkWorker", // Known noisy function + "testing.(*B).ResetTimer" // Test infrastructure +] +``` + +### 5. Set Reasonable Thresholds + +- `min_change_threshold`: 5-10% for most cases +- `max_regression_threshold`: 15-25% for most cases +- Critical paths: 1-5% + +### 6. Monitor and Adjust + +Review CI/CD failures and adjust thresholds based on: + +- False positives (too sensitive) +- Missed regressions (not sensitive enough) +- Team feedback + +## Troubleshooting + +### Common Issues + +1. **Configuration not loaded**: Ensure `config_template.json` is at project root +2. **Functions still causing failures**: Check `ignore_functions` and `ignore_prefixes` +3. **Thresholds not working**: Verify `min_change_threshold` and `max_regression_threshold` +4. **Global vs benchmark settings**: Benchmark-specific settings override global +5. **CLI flags vs config**: When using CI/CD config, `--fail-on-regression` and `--regression-threshold` are optional + +### Debug Information + +Prof logs configuration loading and filtering decisions: + +```bash +prof track auto --base baseline --current PR --bench-name "BenchmarkMyFunction" +``` + +Look for logs like: + +- "Applied CI/CD configuration filtering" +- "Function ignored by CI/CD config" +- "Performance regression below minimum threshold" + +### Validation + +Prof validates configuration on startup. Common validation errors: + +- Negative thresholds +- Malformed JSON + +## Migration from Command-Line + +If you're currently using command-line flags: + +### Before (Command-Line Only) + +```bash +prof track auto --base baseline --current PR \ + --bench-name "BenchmarkMyFunction" \ + --fail-on-regression --regression-threshold 10.0 +``` + +### After (With Configuration) + +```json +{ + "ci_config": { + "global": { + "max_regression_threshold": 10.0 + } + } +} +``` + +The configuration file provides the same functionality with more flexibility and better maintainability. diff --git a/engine/tracker/api.go b/engine/tracker/api.go index 1c92c1b..d613dcb 100644 --- a/engine/tracker/api.go +++ b/engine/tracker/api.go @@ -3,7 +3,11 @@ package tracker import ( "fmt" "log/slog" + "strings" + "math" + + "github.com/AlexsanderHamir/prof/internal" "github.com/AlexsanderHamir/prof/parser" ) @@ -47,11 +51,9 @@ func RunTrackAuto(selections *Selections) error { report.ChooseOutputFormat(selections.OutputFormat) - if selections.UseThreshold && selections.RegressionThreshold > 0.0 { - worst := report.WorstRegression() - if worst != nil && worst.FlatChangePercent >= selections.RegressionThreshold { - return fmt.Errorf("performance regression %.2f%% in %s exceeds threshold %.2f%%", worst.FlatChangePercent, worst.FunctionName, selections.RegressionThreshold) - } + // Apply CI/CD filtering and thresholds + if err = applyCIConfiguration(report, selections); err != nil { + return err } return nil @@ -76,16 +78,213 @@ func RunTrackManual(selections *Selections) error { report.ChooseOutputFormat(selections.OutputFormat) + // Apply CI/CD filtering and thresholds + if err = applyCIConfiguration(report, selections); err != nil { + return err + } + + return nil +} + +// applyCIConfiguration applies CI/CD configuration to the performance report +func applyCIConfiguration(report *ProfileChangeReport, selections *Selections) error { + // Load CI/CD configuration + cfg, err := internal.LoadFromFile(internal.ConfigFilename) + if err != nil { + slog.Info("No CI/CD configuration found, using command-line settings only") + // Fall back to command-line threshold logic + return applyCommandLineThresholds(report, selections) + } + + // Apply CI/CD filtering + report.ApplyCIConfiguration(cfg.CIConfig, selections.BenchmarkName) + + // Check if CLI flags were provided for regression checking + cliFlagsProvided := selections.UseThreshold || selections.RegressionThreshold > 0.0 + + if cliFlagsProvided { + // User provided CLI flags, use them (with CI/CD config as fallback) + return applyCommandLineThresholds(report, selections) + } + + // No CLI flags provided, use CI/CD config only + slog.Info("No CLI regression flags provided, using CI/CD configuration settings") + return applyCICDThresholdsOnly(report, selections, cfg.CIConfig) +} + +// applyCommandLineThresholds applies the legacy command-line threshold logic +func applyCommandLineThresholds(report *ProfileChangeReport, selections *Selections) error { if selections.UseThreshold && selections.RegressionThreshold > 0.0 { worst := report.WorstRegression() if worst != nil && worst.FlatChangePercent >= selections.RegressionThreshold { return fmt.Errorf("performance regression %.2f%% in %s exceeds threshold %.2f%%", worst.FlatChangePercent, worst.FunctionName, selections.RegressionThreshold) } } + return nil +} + +// applyCICDThresholdsOnly applies CI/CD specific threshold logic only, without CLI flags +func applyCICDThresholdsOnly(report *ProfileChangeReport, selections *Selections, cicdConfig *internal.CIConfig) error { + benchmarkName := selections.BenchmarkName + if benchmarkName == "" { + benchmarkName = "unknown" + } + + // Get effective regression threshold + effectiveThreshold := getEffectiveRegressionThreshold(cicdConfig, benchmarkName, 0.0) // Use 0.0 for no CLI threshold + + // Get minimum change threshold + minChangeThreshold := getMinChangeThreshold(cicdConfig, benchmarkName) + + // Check if we should fail on improvements + failOnImprovement := shouldFailOnImprovement(cicdConfig, benchmarkName) + + // Apply thresholds + if effectiveThreshold > 0.0 { + worst := report.WorstRegression() + if worst != nil && worst.FlatChangePercent >= effectiveThreshold { + // Check if function should be ignored by CI/CD config + if !shouldIgnoreFunction(cicdConfig, worst.FunctionName, benchmarkName) { + return fmt.Errorf("performance regression %.2f%% in %s exceeds CI/CD threshold %.2f%%", + worst.FlatChangePercent, worst.FunctionName, effectiveThreshold) + } + } + } + + // Check for improvements if configured to fail on them + if failOnImprovement { + best := report.BestImprovement() + if best != nil && math.Abs(best.FlatChangePercent) >= minChangeThreshold { + if !shouldIgnoreFunction(cicdConfig, best.FunctionName, benchmarkName) { + return fmt.Errorf("unexpected performance improvement %.2f%% in %s (configured to fail on improvements)", + math.Abs(best.FlatChangePercent), best.FunctionName) + } + } + } + + // Check minimum change threshold + if minChangeThreshold > 0.0 { + worst := report.WorstRegression() + if worst != nil && worst.FlatChangePercent < minChangeThreshold { + slog.Info("Performance regression below minimum threshold, not failing CI/CD", + "function", worst.FunctionName, + "change", worst.FlatChangePercent, + "threshold", minChangeThreshold) + } + } return nil } +// Helper functions for CI/CD configuration +func getEffectiveRegressionThreshold(cicdConfig *internal.CIConfig, benchmarkName string, commandLineThreshold float64) float64 { + if cicdConfig == nil { + return commandLineThreshold + } + + // Check benchmark-specific config first + if benchmarkConfig, exists := cicdConfig.Benchmarks[benchmarkName]; exists && benchmarkConfig.MaxRegressionThreshold > 0 { + if commandLineThreshold > 0 { + if commandLineThreshold < benchmarkConfig.MaxRegressionThreshold { + return commandLineThreshold + } + return benchmarkConfig.MaxRegressionThreshold + } + return benchmarkConfig.MaxRegressionThreshold + } + + // Fall back to global config + if cicdConfig.Global != nil && cicdConfig.Global.MaxRegressionThreshold > 0 { + if commandLineThreshold > 0 { + if commandLineThreshold < cicdConfig.Global.MaxRegressionThreshold { + return commandLineThreshold + } + return cicdConfig.Global.MaxRegressionThreshold + } + return cicdConfig.Global.MaxRegressionThreshold + } + + return commandLineThreshold +} + +func getMinChangeThreshold(cicdConfig *internal.CIConfig, benchmarkName string) float64 { + if cicdConfig == nil { + return 0.0 + } + + // Check benchmark-specific config first + if benchmarkConfig, exists := cicdConfig.Benchmarks[benchmarkName]; exists && benchmarkConfig.MinChangeThreshold > 0 { + return benchmarkConfig.MinChangeThreshold + } + + // Fall back to global config + if cicdConfig.Global != nil && cicdConfig.Global.MinChangeThreshold > 0 { + return cicdConfig.Global.MinChangeThreshold + } + + return 0.0 +} + +func shouldFailOnImprovement(cicdConfig *internal.CIConfig, benchmarkName string) bool { + if cicdConfig == nil { + return false + } + + // Check benchmark-specific config first + if benchmarkConfig, exists := cicdConfig.Benchmarks[benchmarkName]; exists { + return benchmarkConfig.FailOnImprovement + } + + // Fall back to global config + if cicdConfig.Global != nil { + return cicdConfig.Global.FailOnImprovement + } + + return false +} + +func shouldIgnoreFunction(cicdConfig *internal.CIConfig, functionName string, benchmarkName string) bool { + if cicdConfig == nil { + return false + } + + // Check benchmark-specific config first + if benchmarkConfig, exists := cicdConfig.Benchmarks[benchmarkName]; exists { + if shouldIgnoreFunctionByConfig(&benchmarkConfig, functionName) { + return true + } + } + + // Fall back to global config + if cicdConfig.Global != nil { + return shouldIgnoreFunctionByConfig(cicdConfig.Global, functionName) + } + + return false +} + +func shouldIgnoreFunctionByConfig(config *internal.CITrackingConfig, functionName string) bool { + if config == nil { + return false + } + + // Check exact function name matches + for _, ignoredFunc := range config.IgnoreFunctions { + if functionName == ignoredFunc { + return true + } + } + + // Check prefix matches + for _, ignoredPrefix := range config.IgnorePrefixes { + if strings.HasPrefix(functionName, ignoredPrefix) { + return true + } + } + + return false +} + // CheckPerformanceDifferences creates the profile report by comparing data from prof's auto run. func CheckPerformanceDifferences(selections *Selections) (*ProfileChangeReport, error) { binFilePathBaseLine, binFilePathCurrent := chooseFileLocations(selections) diff --git a/engine/tracker/profile_change_report.go b/engine/tracker/profile_change_report.go index a60cb08..97561a3 100644 --- a/engine/tracker/profile_change_report.go +++ b/engine/tracker/profile_change_report.go @@ -483,3 +483,55 @@ func (r *ProfileChangeReport) WorstRegression() *FunctionChangeResult { } return worst } + +// BestImprovement returns the single best improvement by absolute flat change percent. +// Returns nil if there are no improvements. +func (r *ProfileChangeReport) BestImprovement() *FunctionChangeResult { + var best *FunctionChangeResult + for _, change := range r.FunctionChanges { + if change.ChangeType != internal.IMPROVEMENT { + continue + } + if best == nil || math.Abs(change.FlatChangePercent) > math.Abs(best.FlatChangePercent) { + best = change + } + } + return best +} + +// ApplyCIConfiguration applies CI/CD configuration filtering to the report +func (r *ProfileChangeReport) ApplyCIConfiguration(cicdConfig *internal.CIConfig, benchmarkName string) { + if cicdConfig == nil { + return + } + + // Get the appropriate CI/CD configuration for this benchmark + var config *internal.CITrackingConfig + if benchmarkConfig, exists := cicdConfig.Benchmarks[benchmarkName]; exists { + config = &benchmarkConfig + } else if cicdConfig.Global != nil { + config = cicdConfig.Global + } else { + return + } + + // Filter out ignored functions + var filteredChanges []*FunctionChangeResult + for _, change := range r.FunctionChanges { + if !shouldIgnoreFunctionByConfig(config, change.FunctionName) { + filteredChanges = append(filteredChanges, change) + } else { + slog.Debug("Function ignored by CI/CD config", + "function", change.FunctionName, + "benchmark", benchmarkName) + } + } + + // Update the report with filtered changes + r.FunctionChanges = filteredChanges + + slog.Info("Applied CI/CD configuration filtering", + "benchmark", benchmarkName, + "original_functions", len(r.FunctionChanges), + "filtered_functions", len(filteredChanges)) +} diff --git a/internal/api.go b/internal/api.go index 8ebb445..a09ac52 100644 --- a/internal/api.go +++ b/internal/api.go @@ -136,6 +136,41 @@ func CreateTemplate() error { IgnoreFunctions: []string{"init", "TestMain", "BenchmarkMain"}, }, }, + CIConfig: &CIConfig{ + Global: &CITrackingConfig{ + // Ignore common noisy functions that shouldn't cause CI/CD failures + IgnoreFunctions: []string{ + "runtime.gcBgMarkWorker", + "runtime.systemstack", + "runtime.mallocgc", + "reflect.ValueOf", + "testing.(*B).launch", + }, + // Ignore runtime and reflect functions that are often noisy + IgnorePrefixes: []string{ + "runtime.", + "reflect.", + "testing.", + }, + // Only fail CI/CD for changes >= 5% + MinChangeThreshold: 5.0, + // Maximum acceptable regression is 15% + MaxRegressionThreshold: 15.0, + // Don't fail on improvements + FailOnImprovement: false, + }, + Benchmarks: map[string]CITrackingConfig{ + "BenchmarkGenPool": { + // Specific settings for this benchmark + IgnoreFunctions: []string{ + "BenchmarkGenPool", + "testing.(*B).ResetTimer", + }, + MinChangeThreshold: 3.0, // More sensitive for this benchmark + MaxRegressionThreshold: 10.0, + }, + }, + }, } if err = os.MkdirAll(filepath.Dir(outputPath), PermDir); err != nil { @@ -144,7 +179,7 @@ func CreateTemplate() error { data, err := json.MarshalIndent(template, "", " ") if err != nil { - return fmt.Errorf("failed to marshal template: %w", err) + return fmt.Errorf("failed to marshal template file: %w", err) } if err = os.WriteFile(outputPath, data, PermFile); err != nil { @@ -153,6 +188,11 @@ func CreateTemplate() error { slog.Info("Template configuration file created", "path", outputPath) slog.Info("Please edit this file with your configuration") + slog.Info("The new CI/CD configuration section allows you to:") + slog.Info(" - Filter out noisy functions from CI/CD failures") + slog.Info(" - Set different thresholds for different benchmarks") + slog.Info(" - Configure severity levels for performance changes") + slog.Info(" - Override command-line regression thresholds") return nil } diff --git a/internal/types.go b/internal/types.go index 1f21a0a..fcbfa8b 100644 --- a/internal/types.go +++ b/internal/types.go @@ -5,6 +5,8 @@ package internal // Config holds the main configuration for the prof tool. type Config struct { FunctionFilter map[string]FunctionFilter `json:"function_collection_filter"` + // CI/CD specific configuration for performance tracking + CIConfig *CIConfig `json:"ci_config,omitempty"` } // FunctionCollectionFilter defines filters for a specific benchmark, @@ -12,7 +14,7 @@ type Config struct { // code line level information for. type FunctionFilter struct { // Prefixes: only collect functions starting with these prefixes - // Example: []string{"github.com/myorg", "main."} + // Example: []string{"github.com/example/GenPool"} IncludePrefixes []string `json:"include_prefixes,omitempty"` // IgnoreFunctions ignores the function name after the last dot. @@ -20,6 +22,37 @@ type FunctionFilter struct { IgnoreFunctions []string `json:"ignore_functions,omitempty"` } +// CIConfig holds CI/CD specific configuration for performance tracking +type CIConfig struct { + // Global CI/CD settings that apply to all tracking operations + Global *CITrackingConfig `json:"global,omitempty"` + + // Benchmark-specific CI/CD settings + Benchmarks map[string]CITrackingConfig `json:"benchmarks,omitempty"` +} + +// CITrackingConfig defines CI/CD specific filtering for performance tracking +type CITrackingConfig struct { + // Functions to ignore during performance comparison (reduces noise) + // These functions won't cause CI/CD failures even if they regress + IgnoreFunctions []string `json:"ignore_functions,omitempty"` + + // Function prefixes to ignore during performance comparison + // Example: ["runtime.", "reflect."] ignores all runtime and reflect functions + IgnorePrefixes []string `json:"ignore_prefixes,omitempty"` + + // Minimum change threshold for CI/CD failure + // Only functions with changes >= this threshold will cause failures + MinChangeThreshold float64 `json:"min_change_threshold,omitempty"` + + // Maximum acceptable regression percentage for CI/CD + // Overrides command-line regression threshold if set + MaxRegressionThreshold float64 `json:"max_regression_threshold,omitempty"` + + // Whether to fail on improvements (useful for detecting unexpected optimizations) + FailOnImprovement bool `json:"fail_on_improvement,omitempty"` +} + // #2 - Function Arguments type LineFilterArgs struct { diff --git a/prof_web_doc/docs/index.md b/prof_web_doc/docs/index.md index e3bac30..0817ba9 100644 --- a/prof_web_doc/docs/index.md +++ b/prof_web_doc/docs/index.md @@ -301,52 +301,227 @@ Flat regression % = (current_time - baseline_time) / baseline_time × 100 **Note:** The threshold applies to **flat time** (time spent directly in the function), not cumulative time (time including all called functions). Flat time gives a more direct measure of the function's own performance impact. -**Important:** Prof commands can be run from the Go project root directory or from within specific package directories. The configuration file (if using one) is always expected at the project root, regardless of where you run the command from, any extra configuration files will be ignored. +## CI/CD Configuration-Based Approach + +Prof now supports a configuration-based approach for CI/CD that eliminates the need for command-line flags and provides more flexibility. + +### Configuration Structure + +Add a `ci_config` section to your existing `config_template.json` file: + +```json +{ + "function_collection_filter": { + // ... existing function filtering ... + }, + "ci_config": { + "global": { + // Global CI/CD settings + }, + "benchmarks": { + "BenchmarkName": { + // Benchmark-specific CI/CD settings + } + } + } +} +``` + +### Global Configuration + +```json +"global": { + "ignore_functions": ["runtime.gcBgMarkWorker", "testing.(*B).ResetTimer"], + "ignore_prefixes": ["runtime.", "reflect.", "testing."], + "min_change_threshold": 5.0, + "max_regression_threshold": 20.0, + "fail_on_improvement": false +} +``` + +### Benchmark-Specific Configuration + +```json +"benchmarks": { + "BenchmarkMyFunction": { + "min_change_threshold": 3.0, + "max_regression_threshold": 10.0 + } +} +``` + +### Function Filtering + +**Ignore specific functions:** + +```json +"ignore_functions": ["runtime.gcBgMarkWorker", "testing.(*B).ResetTimer"] +``` + +**Ignore function prefixes:** + +```json +"ignore_prefixes": ["runtime.", "reflect.", "testing."] +``` + +### Threshold Configuration + +- `min_change_threshold`: Minimum change % to trigger CI/CD failure +- `max_regression_threshold`: Maximum acceptable regression % +- Command-line flags are optional when using configuration + +### Complete Example + +```json +{ + "ci_config": { + "global": { + "ignore_prefixes": ["runtime.", "reflect.", "testing."], + "min_change_threshold": 5.0, + "max_regression_threshold": 20.0 + }, + "benchmarks": { + "BenchmarkCriticalPath": { + "min_change_threshold": 1.0, + "max_regression_threshold": 5.0 + } + } + } +} +``` + +### CI/CD Integration + +With configuration-based CI/CD, you no longer need `--fail-on-regression` or `--regression-threshold` flags: ```bash -prof track auto \ - --base baseline \ - --current PR \ - --profile-type cpu \ - --bench-name BenchmarkGenPool \ - --output-format summary \ - --fail-on-regression \ - --regression-threshold 5.0 +prof track auto --base baseline --current PR \ + --profile-type cpu --bench-name "BenchmarkMyFunction" \ ``` -**Example GitHub Actions job:** +**Example GitHub Actions:** ```yaml -name: perf-regression-check -on: [pull_request] -jobs: - check: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-go@v5 - with: - go-version: ">=1.24" - - name: Install prof - run: go install github.com/AlexsanderHamir/prof/cmd/prof@latest - - name: Collect baseline (main) - run: | - git fetch origin main --depth=1 - git checkout -qf origin/main - # prof can be run from the Go project root directory - cd ${{ github.workspace }} - prof auto --benchmarks "BenchmarkGenPool" --profiles "cpu" --count 5 --tag baseline - - name: Collect current (PR) - run: | - git checkout - - # prof can be run from the Go project root directory - cd ${{ github.workspace }} - prof auto --benchmarks "BenchmarkGenPool" --profiles "cpu" --count 5 --tag PR - - name: Compare and fail on regression - run: | - # prof can be run from the Go project root directory - cd ${{ github.workspace }} - prof track auto --base baseline --current PR \ - --profile-type cpu --bench-name "BenchmarkGenPool" \ - --output-format summary --fail-on-regression --regression-threshold 5.0 +- name: Check for regressions + run: | + prof track auto --base baseline --current PR \ + --profile-type cpu --bench-name "BenchmarkMyFunction" \ +``` + +**Configuration File Location:** Must be at project root (same directory as `go.mod`). + +# Prof Tools + +Prof provides additional tools that can easily operate on the collected data for enhanced analysis and visualization. + +## Tools Overview + +The `prof tools` command provides access to specialized analysis tools: + +```bash +prof tools [command] [flags] +``` + +Available tools: + +- **`benchstat`**: Statistical analysis of benchmark results +- **`qcachegrind`**: Visual call graph analysis + +## Benchstat Tool + +Runs Go's official `benchstat` command on collected benchmark data. + +### Usage + +```bash +prof tools benchstat --base --current --bench-name +``` + +### Example + +```bash +prof tools benchstat --base baseline --current optimized --bench-name BenchmarkGenPool +``` + +### Prerequisites + +```bash +go install golang.org/x/perf/cmd/benchstat@latest +``` + +### Output + +Results are saved to `bench/tools/benchstats/{benchmark_name}_results.txt` + +## QCacheGrind Tool + +Generates call graph data from binary profile files and launches the QCacheGrind visualizer. + +### Usage + +```bash +prof tools qcachegrind --tag --profiles --bench-name +``` + +### Example + +```bash +prof tools qcachegrind --tag optimized --profiles cpu --bench-name BenchmarkGenPool +``` + +### Prerequisites + +**Ubuntu/Debian:** + +```bash +sudo apt-get install qcachegrind +``` + +**macOS:** + +```bash +brew install qcachegrind +``` + +### Output + +Callgrind files are saved to `bench/tools/qcachegrind/{benchmark_name}_{profile_type}.callgrind` + +## Tool Output Organization + +``` +bench/ +├── baseline/ +├── optimized/ +└── tools/ + ├── benchstats/ + │ └── BenchmarkGenPool_results.txt + └── qcachegrind/ + └── BenchmarkGenPool_cpu.callgrind +``` + +## Integration with Existing Workflow + +1. **Collect data**: Use `prof auto` or `prof tui` +2. **Compare performance**: Use `prof track` +3. **Deep analysis**: Use `prof tools` +4. **Visual exploration**: Use QCacheGrind for interactive call graph analysis + +## Best Practices + +**Combine tools for comprehensive analysis:** + +```bash +# Collect data +prof auto --benchmarks "BenchmarkGenPool" --profiles "cpu,memory" --count 10 --tag baseline +prof auto --benchmarks "BenchmarkGenPool" --profiles "cpu,memory" --count 10 --tag optimized + +# Compare performance +prof track auto --base baseline --current optimized --bench-name BenchmarkGenPool + +# Statistical validation +prof tools benchstat --base baseline --current optimized --bench-name BenchmarkGenPool + +# Deep analysis +prof tools qcachegrind --tag optimized --profiles cpu --bench-name BenchmarkGenPool ``` diff --git a/readme.md b/readme.md index c36e406..e0b0f47 100644 --- a/readme.md +++ b/readme.md @@ -11,7 +11,7 @@ Prof automates Go performance profiling by collecting all pprof data in one comm ![Version](https://img.shields.io/github/v/tag/AlexsanderHamir/prof?sort=semver) ![Go Version](https://img.shields.io/badge/Go-1.24.3%2B-blue) -📖 [Documentation](https://alexsanderhamir.github.io/prof/) | ▶️ [Watch Demo Video](https://cdn.jsdelivr.net/gh/AlexsanderHamir/assets@main/output.mp4) | ▶️ [Watch TUI Demo](https://cdn.jsdelivr.net/gh/AlexsanderHamir/assets@main/tui_prof.mp4) | 🌟 [Project Vision](#project-vision) +📖 [Documentation](https://alexsanderhamir.github.io/prof/) | ▶️ [Watch Demo Video](https://cdn.jsdelivr.net/gh/AlexsanderHamir/assets@main/output.mp4) | ▶️ [Watch TUI Demo](https://cdn.jsdelivr.net/gh/AlexsanderHamir/assets@main/tui_prof.mp4) ## Why Prof? @@ -59,6 +59,8 @@ Fail CI/CD pipelines on performance regressions with configurable thresholds: prof track auto --base "baseline" --current "PR" --profile-type "cpu" --bench-name "BenchmarkName" --fail-on-regression --regression-threshold 5.0 ``` +**Enhanced CI/CD Support**: Configure function filtering, and custom thresholds to reduce noise and make CI/CD more reliable. See [CI/CD Configuration Guide](docs/cicd_configuration.md) for details. + ### 📁 Organized Output All profiling data is automatically organized under `bench//` directories with clear structure. @@ -119,20 +121,6 @@ prof track auto --base "baseline" --current "optimized" --profile-type "cpu" --b - Install graphviz - A Go module (`go.mod`) at the repository root -## Project Vision - -Go developers today juggle too many tools (`pprof`, `benchstat`, `qcachegrind`, `go tool trace`, custom scripts) just to understand performance. Each has its own workflow, and connecting them takes time and expertise. - -`prof` is building a different path: - -- **One hub** — unify the best profiling and benchmarking tools under a single interface. -- **Smooth workflows** — make profiling as quick and natural as running tests. -- **Open platform** — allow extensions and custom tools, so anyone can shape how performance is analyzed. - -The long-term vision: **turn `prof` into the go-to performance analysis hub for Go — one install, every workflow, no friction.** - -As a contributor, you’re not just fixing bugs — you’re helping design the future of how Go developers improve their code. - ## License This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.