cscli hubtest: better report docker/nuclei errors (#4052)

mmetc · web-flow · commit 9662e5fb661e · 2025-11-18T11:23:01.000+01:00
* cscli hubtest: better report docker/nuclei errors

* lint (print, space)
diff --git a/cmd/crowdsec-cli/clihubtest/clean.go b/cmd/crowdsec-cli/clihubtest/clean.go
@@ -3,6 +3,7 @@ package clihubtest
 import (
 	"errors"
 	"fmt"
+	"os"
 
 	"github.com/spf13/cobra"
 
@@ -21,7 +22,7 @@ func (*cliHubTest) newCleanCmd() *cobra.Command {
 				return errors.New("please provide test to run or --all flag")
 			}
 
-			fmt.Println("Cleaning test data...")
+			fmt.Fprintln(os.Stdout, "Cleaning test data...")
 
 			tests := []*hubtest.HubTestItem{}
 
@@ -37,6 +38,7 @@ func (*cliHubTest) newCleanCmd() *cobra.Command {
 					if err != nil {
 						return fmt.Errorf("unable to load test '%s': %w", testName, err)
 					}
+
 					tests = append(tests, test)
 				}
 			}
diff --git a/cmd/crowdsec-cli/clihubtest/coverage.go b/cmd/crowdsec-cli/clihubtest/coverage.go
@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"math"
+	"os"
 
 	"github.com/fatih/color"
 	"github.com/spf13/cobra"
@@ -76,11 +77,11 @@ func (cli *cliHubTest) coverage(showScenarioCov bool, showParserCov bool, showAp
 	if showOnlyPercent {
 		switch {
 		case showParserCov:
-			fmt.Printf("parsers=%d%%", parserCoveragePercent)
+			fmt.Fprintf(os.Stdout, "parsers=%d%%", parserCoveragePercent)
 		case showScenarioCov:
-			fmt.Printf("scenarios=%d%%", scenarioCoveragePercent)
+			fmt.Fprintf(os.Stdout, "scenarios=%d%%", scenarioCoveragePercent)
 		case showAppsecCov:
-			fmt.Printf("appsec_rules=%d%%", appsecRuleCoveragePercent)
+			fmt.Fprintf(os.Stdout, "appsec_rules=%d%%", appsecRuleCoveragePercent)
 		}
 
 		return nil
@@ -100,40 +101,40 @@ func (cli *cliHubTest) coverage(showScenarioCov bool, showParserCov bool, showAp
 			hubTestCoverageTable(color.Output, cfg.Cscli.Color, []string{"Appsec Rule", "Status", "Number of tests"}, parserCoverage)
 		}
 
-		fmt.Println()
+		fmt.Fprintln(os.Stdout)
 
 		if showParserCov {
-			fmt.Printf("PARSERS    : %d%% of coverage\n", parserCoveragePercent)
+			fmt.Fprintf(os.Stdout, "PARSERS    : %d%% of coverage\n", parserCoveragePercent)
 		}
 
 		if showScenarioCov {
-			fmt.Printf("SCENARIOS  : %d%% of coverage\n", scenarioCoveragePercent)
+			fmt.Fprintf(os.Stdout, "SCENARIOS  : %d%% of coverage\n", scenarioCoveragePercent)
 		}
 
 		if showAppsecCov {
-			fmt.Printf("APPSEC RULES  : %d%% of coverage\n", appsecRuleCoveragePercent)
+			fmt.Fprintf(os.Stdout, "APPSEC RULES  : %d%% of coverage\n", appsecRuleCoveragePercent)
 		}
 	case "json":
 		dump, err := json.MarshalIndent(parserCoverage, "", " ")
 		if err != nil {
 			return err
 		}
 
-		fmt.Printf("%s", dump)
+		fmt.Fprint(os.Stdout, string(dump))
 
 		dump, err = json.MarshalIndent(scenarioCoverage, "", " ")
 		if err != nil {
 			return err
 		}
 
-		fmt.Printf("%s", dump)
+		fmt.Fprint(os.Stdout, string(dump))
 
 		dump, err = json.MarshalIndent(appsecRuleCoverage, "", " ")
 		if err != nil {
 			return err
 		}
 
-		fmt.Printf("%s", dump)
+		fmt.Fprint(os.Stdout, string(dump))
 	default:
 		return errors.New("only human/json output modes are supported")
 	}
diff --git a/cmd/crowdsec-cli/clihubtest/create.go b/cmd/crowdsec-cli/clihubtest/create.go
@@ -36,6 +36,7 @@ cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios
 		RunE: func(_ *cobra.Command, args []string) error {
 			testName := args[0]
 			testPath := filepath.Join(hubPtr.HubTestPath, testName)
+
 			if _, err := os.Stat(testPath); os.IsExist(err) {
 				return fmt.Errorf("test '%s' already exists in '%s', exiting", testName, testPath)
 			}
@@ -53,8 +54,8 @@ cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios
 			}
 
 			configFilePath := filepath.Join(testPath, "config.yaml")
-
 			configFileData := &hubtest.HubTestItemConfig{}
+
 			if logType == "appsec" {
 				// create empty nuclei template file
 				nucleiFileName := testName + ".yaml"
@@ -69,12 +70,16 @@ cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios
 				if ntpl == nil {
 					return errors.New("unable to parse nuclei template")
 				}
+
 				if err := ntpl.ExecuteTemplate(nucleiFile, "nuclei", struct{ TestName string }{TestName: testName}); err != nil {
 					return fmt.Errorf("executing nuclei template %s: %w", nucleiFile.Name(), err)
 				}
+
 				nucleiFile.Close()
+
 				configFileData.AppsecRules = []string{"./appsec-rules/<author>/your_rule_here.yaml"}
 				configFileData.NucleiTemplate = nucleiFileName
+
 				fmt.Fprintln(os.Stdout)
 				fmt.Fprintf(os.Stdout, "  Test name                   :  %s\n", testName)
 				fmt.Fprintf(os.Stdout, "  Test path                   :  %s\n", testPath)
@@ -84,25 +89,32 @@ cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios
 				// create empty log file
 				logFileName := testName + ".log"
 				logFilePath := filepath.Join(testPath, logFileName)
+
 				logFile, err := os.Create(logFilePath)
 				if err != nil {
 					return err
 				}
+
 				logFile.Close()
 
 				// create empty parser assertion file
 				parserAssertFilePath := filepath.Join(testPath, hubtest.ParserAssertFileName)
+
 				parserAssertFile, err := os.Create(parserAssertFilePath)
 				if err != nil {
 					return err
 				}
+
 				parserAssertFile.Close()
+
 				// create empty scenario assertion file
 				scenarioAssertFilePath := filepath.Join(testPath, hubtest.ScenarioAssertFileName)
+
 				scenarioAssertFile, err := os.Create(scenarioAssertFilePath)
 				if err != nil {
 					return err
 				}
+
 				scenarioAssertFile.Close()
 
 				parsers = append(parsers, "crowdsecurity/syslog-logs")
@@ -115,13 +127,15 @@ cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios
 				if len(postoverflows) == 0 {
 					postoverflows = append(postoverflows, "")
 				}
+
 				configFileData.Parsers = parsers
 				configFileData.Scenarios = scenarios
 				configFileData.PostOverflows = postoverflows
 				configFileData.LogFile = logFileName
 				configFileData.LogType = logType
 				configFileData.IgnoreParsers = ignoreParsers
 				configFileData.Labels = labels
+
 				fmt.Fprintln(os.Stdout)
 				fmt.Fprintf(os.Stdout, "  Test name                   :  %s\n", testName)
 				fmt.Fprintf(os.Stdout, "  Test path                   :  %s\n", testPath)
@@ -135,14 +149,17 @@ cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios
 			if err != nil {
 				return fmt.Errorf("open: %w", err)
 			}
+
 			data, err := yaml.Marshal(configFileData)
 			if err != nil {
 				return fmt.Errorf("serialize: %w", err)
 			}
+
 			_, err = fd.Write(data)
 			if err != nil {
 				return fmt.Errorf("write: %w", err)
 			}
+
 			if err := fd.Close(); err != nil {
 				return fmt.Errorf("close: %w", err)
 			}
diff --git a/cmd/crowdsec-cli/clihubtest/eval.go b/cmd/crowdsec-cli/clihubtest/eval.go
@@ -2,6 +2,7 @@ package clihubtest
 
 import (
 	"fmt"
+	"os"
 
 	"github.com/spf13/cobra"
 
@@ -33,7 +34,7 @@ func (*cliHubTest) newEvalCmd() *cobra.Command {
 					return err
 				}
 
-				fmt.Print(output)
+				fmt.Fprint(os.Stdout, output)
 			}
 
 			return nil
diff --git a/cmd/crowdsec-cli/clihubtest/hubtest.go b/cmd/crowdsec-cli/clihubtest/hubtest.go
@@ -42,6 +42,7 @@ func (cli *cliHubTest) NewCommand() *cobra.Command {
 		DisableAutoGenTag: true,
 		PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
 			var err error
+
 			HubTest, err = hubtest.NewHubTest(hubPath, crowdsecPath, cscliPath, false)
 			if err != nil {
 				return fmt.Errorf("unable to load hubtest: %+v", err)
diff --git a/cmd/crowdsec-cli/clihubtest/info.go b/cmd/crowdsec-cli/clihubtest/info.go
@@ -2,6 +2,7 @@ package clihubtest
 
 import (
 	"fmt"
+	"os"
 	"path/filepath"
 	"strings"
 
@@ -23,18 +24,21 @@ func (*cliHubTest) newInfoCmd() *cobra.Command {
 				if err != nil {
 					return fmt.Errorf("unable to load test '%s': %w", testName, err)
 				}
-				fmt.Println()
-				fmt.Printf("  Test name                   :  %s\n", test.Name)
-				fmt.Printf("  Test path                   :  %s\n", test.Path)
+
+				fmt.Fprintln(os.Stdout)
+				fmt.Fprintf(os.Stdout, "  Test name                   :  %s\n", test.Name)
+				fmt.Fprintf(os.Stdout, "  Test path                   :  %s\n", test.Path)
+
 				if isAppsecTest {
-					fmt.Printf("  Nuclei Template             :  %s\n", test.Config.NucleiTemplate)
-					fmt.Printf("  Appsec Rules                  :  %s\n", strings.Join(test.Config.AppsecRules, ", "))
+					fmt.Fprintf(os.Stdout, "  Nuclei Template             :  %s\n", test.Config.NucleiTemplate)
+					fmt.Fprintf(os.Stdout, "  Appsec Rules                  :  %s\n", strings.Join(test.Config.AppsecRules, ", "))
 				} else {
-					fmt.Printf("  Log file                    :  %s\n", filepath.Join(test.Path, test.Config.LogFile))
-					fmt.Printf("  Parser assertion file       :  %s\n", filepath.Join(test.Path, hubtest.ParserAssertFileName))
-					fmt.Printf("  Scenario assertion file     :  %s\n", filepath.Join(test.Path, hubtest.ScenarioAssertFileName))
+					fmt.Fprintf(os.Stdout, "  Log file                    :  %s\n", filepath.Join(test.Path, test.Config.LogFile))
+					fmt.Fprintf(os.Stdout, "  Parser assertion file       :  %s\n", filepath.Join(test.Path, hubtest.ParserAssertFileName))
+					fmt.Fprintf(os.Stdout, "  Scenario assertion file     :  %s\n", filepath.Join(test.Path, hubtest.ScenarioAssertFileName))
 				}
-				fmt.Printf("  Configuration File          :  %s\n", filepath.Join(test.Path, "config.yaml"))
+
+				fmt.Fprintf(os.Stdout, "  Configuration File          :  %s\n", filepath.Join(test.Path, "config.yaml"))
 			}
 
 			return nil
diff --git a/cmd/crowdsec-cli/clihubtest/list.go b/cmd/crowdsec-cli/clihubtest/list.go
@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"os"
 
 	"github.com/fatih/color"
 	"github.com/spf13/cobra"
@@ -32,7 +33,8 @@ func (cli *cliHubTest) newListCmd() *cobra.Command {
 				if err != nil {
 					return err
 				}
-				fmt.Println(string(j))
+
+				fmt.Fprintln(os.Stdout, string(j))
 			default:
 				return errors.New("only human/json output modes are supported")
 			}
diff --git a/cmd/crowdsec-cli/clihubtest/run.go b/cmd/crowdsec-cli/clihubtest/run.go
@@ -47,7 +47,7 @@ func (cli *cliHubTest) run(ctx context.Context, all bool, nucleiTargetHost strin
 	eg, gctx := errgroup.WithContext(ctx)
 
 	if isAppsecTest {
-		log.Info("Appsec tests can not run in parallel: setting max_jobs=1")
+		fmt.Fprintln(os.Stdout, "Appsec tests can not run in parallel: setting max_jobs=1")
 
 		maxJobs = 1
 	}
diff --git a/cmd/crowdsec-cli/clihubtest/table.go b/cmd/crowdsec-cli/clihubtest/table.go
@@ -3,6 +3,7 @@ package clihubtest
 import (
 	"fmt"
 	"io"
+	"os"
 	"strconv"
 
 	"github.com/jedib0t/go-pretty/v6/text"
@@ -35,7 +36,7 @@ func hubTestResultTable(out io.Writer, wantColor string, testMap map[string]*hub
 	if showTable {
 		t.Render()
 	} else {
-		fmt.Println("All tests passed, use --report-success for more details.")
+		fmt.Fprintln(os.Stdout, "All tests passed, use --report-success for more details.")
 	}
 }
 
diff --git a/pkg/hubtest/nucleirunner.go b/pkg/hubtest/nucleirunner.go
@@ -29,6 +29,8 @@ func (nc *NucleiConfig) RunNucleiTemplate(ctx context.Context, testName string,
 	outputPrefix := fmt.Sprintf("%s/%s-%d", nc.OutputDir, testName, tstamp)
 	// CVE-2023-34362_CVE-2023-34362-1702562399_stderr.txt
 	args := []string{
+		// removing the banner with --silent also removes useful [WRN] lines, so it is what it is
+		// "-silent",
 		"-u", target,
 		"-t", templatePath,
 		"-o", outputPrefix + ".json",
@@ -46,25 +48,33 @@ func (nc *NucleiConfig) RunNucleiTemplate(ctx context.Context, testName string,
 	cmd.Stdout = &out
 	cmd.Stderr = &outErr
 
-	err := cmd.Run()
+	cmdErr := cmd.Run()
 	if err := os.WriteFile(outputPrefix+"_stdout.txt", out.Bytes(), 0o644); err != nil {
-		log.Warningf("Error writing stdout: %s", err)
+		log.Errorf("Error writing stdout: %s", err)
 	}
 
-	if err := os.WriteFile(outputPrefix+"_stderr.txt", outErr.Bytes(), 0o644); err != nil {
-		log.Warningf("Error writing stderr: %s", err)
+	errBytes := outErr.Bytes()
+
+	if err := os.WriteFile(outputPrefix+"_stderr.txt", errBytes, 0o644); err != nil {
+		log.Errorf("Error writing stderr: %s", err)
 	}
 
-	if err != nil {
-		log.Warningf("Stdout saved to %s", outputPrefix+"_stdout.txt")
-		log.Warningf("Stderr saved to %s", outputPrefix+"_stderr.txt")
-		log.Warningf("Nuclei generated output saved to %s", outputPrefix+".json")
+	if cmdErr != nil {
+		// display stderr in addition to writing to a file
+		os.Stdout.Write(errBytes)
+		os.Stdout.WriteString("\n")
+
+		fmt.Fprintln(os.Stdout, "Stdout saved to", outputPrefix+"_stdout.txt")
+		fmt.Fprintln(os.Stdout, "Stderr saved to", outputPrefix+"_stderr.txt")
+		fmt.Fprintln(os.Stdout, "Nuclei generated output saved to", outputPrefix+".json")
+
+		return fmt.Errorf("%w: %v", ErrNucleiRunFail, cmdErr)
+	}
 
-		return fmt.Errorf("%w: %v", ErrNucleiRunFail, err)
-	} else if out.String() == "" {
-		log.Warningf("Stdout saved to %s", outputPrefix+"_stdout.txt")
-		log.Warningf("Stderr saved to %s", outputPrefix+"_stderr.txt")
-		log.Warningf("Nuclei generated output saved to %s", outputPrefix+".json")
+	if out.String() == "" {
+		fmt.Fprintln(os.Stdout, "Stdout saved to", outputPrefix+"_stdout.txt")
+		fmt.Fprintln(os.Stdout, "Stderr saved to", outputPrefix+"_stderr.txt")
+		fmt.Fprintln(os.Stdout, "Nuclei generated output saved to", outputPrefix+".json")
 
 		// No stdout means no finding, it means our test failed
 		return ErrNucleiTemplateFail
diff --git a/pkg/hubtest/utils.go b/pkg/hubtest/utils.go
@@ -18,13 +18,13 @@ func IsAlive(ctx context.Context, target string) (bool, error) {
 		dialer := &net.Dialer{}
 		conn, err := dialer.DialContext(ctx, "tcp", target)
 		if err == nil {
-			log.Debugf("'%s' is up after %s", target, time.Since(start))
+			log.Debugf("%q is up after %s", target, time.Since(start))
 			conn.Close()
 			return true, nil
 		}
 		time.Sleep(500 * time.Millisecond)
 		if time.Since(start) > 10*time.Second {
-			return false, fmt.Errorf("took more than 10s for %s to be available", target)
+			return false, fmt.Errorf(`timeout connecting to %s -- did you run "sudo docker compose -f docker/appsec/docker-compose.yaml up -d --build" ?`, target)
 		}
 	}
 }

Original file line number	Diff line number	Diff line change
`@@ -3,6 +3,7 @@ package clihubtest`
`3`	`3`	`import (`
`4`	`4`	`"errors"`
`5`	`5`	`"fmt"`
	`6`	`+ "os"`
`6`	`7`
`7`	`8`	`"github.com/spf13/cobra"`
`8`	`9`
`@@ -21,7 +22,7 @@ func (cliHubTest) newCleanCmd() cobra.Command {`
`21`	`22`	`return errors.New("please provide test to run or --all flag")`
`22`	`23`	`}`
`23`	`24`
`24`		`- fmt.Println("Cleaning test data...")`
	`25`	`+ fmt.Fprintln(os.Stdout, "Cleaning test data...")`
`25`	`26`
`26`	`27`	`tests := []*hubtest.HubTestItem{}`
`27`	`28`
`@@ -37,6 +38,7 @@ func (cliHubTest) newCleanCmd() cobra.Command {`
`37`	`38`	`if err != nil {`
`38`	`39`	`return fmt.Errorf("unable to load test '%s': %w", testName, err)`
`39`	`40`	`}`
	`41`	`+`
`40`	`42`	`tests = append(tests, test)`
`41`	`43`	`}`
`42`	`44`	`}`
Original file line number	Diff line number	Diff line change
`@@ -2,6 +2,7 @@ package clihubtest`
`2`	`2`
`3`	`3`	`import (`
`4`	`4`	`"fmt"`
	`5`	`+ "os"`
`5`	`6`
`6`	`7`	`"github.com/spf13/cobra"`
`7`	`8`
`@@ -33,7 +34,7 @@ func (cliHubTest) newEvalCmd() cobra.Command {`
`33`	`34`	`return err`
`34`	`35`	`}`
`35`	`36`
`36`		`- fmt.Print(output)`
	`37`	`+ fmt.Fprint(os.Stdout, output)`
`37`	`38`	`}`
`38`	`39`
`39`	`40`	`return nil`
Original file line number	Diff line number	Diff line change
`@@ -47,7 +47,7 @@ func (cli *cliHubTest) run(ctx context.Context, all bool, nucleiTargetHost strin`
`47`	`47`	`eg, gctx := errgroup.WithContext(ctx)`
`48`	`48`
`49`	`49`	`if isAppsecTest {`
`50`		`- log.Info("Appsec tests can not run in parallel: setting max_jobs=1")`
	`50`	`+ fmt.Fprintln(os.Stdout, "Appsec tests can not run in parallel: setting max_jobs=1")`
`51`	`51`
`52`	`52`	`maxJobs = 1`
`53`	`53`	`}`