refact pkg/parser: ExtraField / Static (#3913)

* refact parser: rename ExtraField -> Static

* refact parser: split Static - RuntimeStatic

* refact parser: extract RuntimeStatic.Apply()

* lint

Author: mmetc

cmd/crowdsec-cli/clilapi/context.go

@@ -3,6 +3,7 @@ package clilapi
 import (
 	"errors"
 	"fmt"
+	"os"
 	"slices"
 	"sort"
 	"strings"
@@ -114,7 +115,7 @@ func (cli *cliLapi) newContextStatusCmd() *cobra.Command {
 			}
 
 			if len(cfg.Crowdsec.ContextToSend) == 0 {
-				fmt.Println("No context found on this agent. You can use 'cscli lapi context add' to add context to your alerts.")
+				fmt.Fprintln(os.Stdout, "No context found on this agent. You can use 'cscli lapi context add' to add context to your alerts.")
 				return nil
 			}
 
@@ -123,7 +124,7 @@ func (cli *cliLapi) newContextStatusCmd() *cobra.Command {
 				return fmt.Errorf("unable to show context status: %w", err)
 			}
 
-			fmt.Print(string(dump))
+			fmt.Fprint(os.Stdout, string(dump))
 
 			return nil
 		},
@@ -185,11 +186,10 @@ cscli lapi context detect crowdsecurity/sshd-logs
 				}
 			}
 
-			fmt.Printf("Acquisition :\n\n")
-			fmt.Printf("  - evt.Line.Module\n")
-			fmt.Printf("  - evt.Line.Raw\n")
-			fmt.Printf("  - evt.Line.Src\n")
-			fmt.Println()
+			fmt.Fprint(os.Stdout, "Acquisition :\n\n")
+			fmt.Fprint(os.Stdout, "  - evt.Line.Module\n")
+			fmt.Fprint(os.Stdout, "  - evt.Line.Raw\n")
+			fmt.Fprint(os.Stdout, "  - evt.Line.Src\n\n")
 
 			parsersKey := make([]string, 0)
 			for k := range fieldByParsers {
@@ -201,13 +201,13 @@ cscli lapi context detect crowdsecurity/sshd-logs
 				if len(fieldByParsers[k]) == 0 {
 					continue
 				}
-				fmt.Printf("%s :\n\n", k)
+				fmt.Fprintf(os.Stdout, "%s :\n\n", k)
 				values := fieldByParsers[k]
 				sort.Strings(values)
 				for _, value := range values {
-					fmt.Printf("  - %s\n", value)
+					fmt.Fprintf(os.Stdout, "  - %s\n", value)
 				}
-				fmt.Println()
+				fmt.Fprintln(os.Stdout)
 			}
 
 			if len(args) > 0 {
@@ -270,7 +270,7 @@ func (cli *cliLapi) newContextCmd() *cobra.Command {
 	return cmd
 }
 
-func detectStaticField(grokStatics []parser.ExtraField) []string {
+func detectStaticField(grokStatics []parser.Static) []string {
 	ret := make([]string, 0)
 
 	for _, static := range grokStatics {

pkg/hubtest/hubtest_item.go

@@ -35,7 +35,7 @@ type HubTestItemConfig struct {
 	LogType               string              `yaml:"log_type,omitempty"`
 	Labels                map[string]string   `yaml:"labels,omitempty"`
 	IgnoreParsers         bool                `yaml:"ignore_parsers,omitempty"`   // if we test a scenario, we don't want to assert on Parser
-	OverrideStatics       []parser.ExtraField `yaml:"override_statics,omitempty"` // Allow to override statics. Executed before s00
+	OverrideStatics       []parser.Static     `yaml:"override_statics,omitempty"` // Allow to override statics. Executed before s00
 	OwnDataDir            bool                `yaml:"own_data_dir,omitempty"`     // Don't share dataDir with the other tests
 }
 
@@ -290,26 +290,34 @@ func (*HubTestItem) ImprovedLogDisplay(crowdsecLogFile string) error {
 	if err != nil {
 		log.Errorf("unable to read crowdsec log file '%s': %s", crowdsecLogFile, err)
 	}
+
 	success := []string{}
 	failures := []string{}
 	general := []string{}
+
 	for _, line := range strings.Split(string(crowdsecLog), "\n") {
 		if strings.Contains(line, `"Evaluating operator: MATCH"`) {
 			success = append(success, line)
 		} else if strings.Contains(line, `"Evaluating operator: NO MATCH"`) {
 			failures = append(failures, line)
 		}
+
 		general = append(general, line)
 	}
+
 	log.Infof("General log -------------\n%s\n", strings.Join(general, "\n"))
+
 	if len(success) > 0 {
 		log.Infof("Success log -------------")
+
 		for _, line := range success {
 			log.Infof("%s - %s", emoji.GreenCircle, line)
 		}
 	}
+
 	if len(failures) > 0 {
 		log.Errorf("Failure log -------------")
+
 		for _, line := range failures {
 			log.Infof("%s - %s", emoji.RedCircle, line)
 		}
@@ -335,7 +343,7 @@ func (t *HubTestItem) RunWithNucleiTemplate(ctx context.Context) error {
 	output, err := cscliRegisterCmd.CombinedOutput()
 	if err != nil {
 		if !strings.Contains(string(output), "unable to create machine: user 'testMachine': user already exist") {
-			fmt.Println(string(output))
+			fmt.Fprintln(os.Stdout, string(output))
 			return fmt.Errorf("fail to run '%s' for test '%s': %w", cscliRegisterCmd.String(), t.Name, err)
 		}
 	}

pkg/parser/grok.go

@@ -0,0 +1,107 @@
+package parser
+
+import (
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/expr-lang/expr"
+	"github.com/expr-lang/expr/vm"
+
+	"github.com/crowdsecurity/grokky"
+	log "github.com/sirupsen/logrus"
+
+	"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
+	"github.com/crowdsecurity/crowdsec/pkg/types"
+)
+
+type Static struct {
+	// if the target is indicated by name Struct.Field etc,
+	TargetByName string `yaml:"target,omitempty"`
+	// if the target field is in Event map
+	Parsed string `yaml:"parsed,omitempty"`
+	// if the target field is in Meta map
+	Meta string `yaml:"meta,omitempty"`
+	// if the target field is in Enriched map
+	Enriched string `yaml:"enriched,omitempty"`
+	// the source is a static value
+	Value string `yaml:"value,omitempty"`
+	// or the result of an Expression
+	ExpValue     string      `yaml:"expression,omitempty"`
+	// or an enrichment method
+	Method string `yaml:"method,omitempty"`
+}
+
+type RuntimeStatic struct {
+	Config *Static
+	RunTimeValue *vm.Program
+}
+
+func (s *Static) Validate(ectx EnricherCtx) error {
+	if s.Method != "" {
+		if s.Value == "" && s.ExpValue == "" {
+			return errors.New("when method is set, expression must be present")
+		}
+
+		if _, ok := ectx.Registered[s.Method]; !ok {
+			log.Warningf("the method %q doesn't exist or the plugin has not been initialized", s.Method)
+		}
+
+		return nil
+	}
+
+	if s.Meta == "" && s.Parsed == "" && s.TargetByName == "" {
+		return errors.New("at least one of meta/event/target must be set")
+	}
+
+	if s.Value == "" && s.ExpValue == "" {
+		return errors.New("value or expression must be set")
+	}
+
+	return nil
+}
+
+func (s *Static) Compile() (*RuntimeStatic, error) {
+	cs := &RuntimeStatic{Config: s}
+
+	if s.ExpValue != "" {
+		prog, err := expr.Compile(s.ExpValue,
+			exprhelpers.GetExprOptions(map[string]any{"evt": &types.Event{}})...)
+		if err != nil {
+			return nil, fmt.Errorf("compiling static expression %q: %w", s.ExpValue, err)
+		}
+
+		cs.RunTimeValue = prog
+	}
+
+	return cs, nil
+}
+
+type GrokPattern struct {
+	// the field to which regexp is going to apply
+	TargetField string `yaml:"apply_on,omitempty"`
+	// the grok/regexp by name (loaded from patterns/*)
+	RegexpName string `yaml:"name,omitempty"`
+	// a proper grok pattern
+	RegexpValue string `yaml:"pattern,omitempty"`
+	// the runtime form of regexpname / regexpvalue
+	RunTimeRegexp grokky.Pattern `yaml:"-"` // the actual regexp
+	// the output of the expression is going to be the source for regexp
+	ExpValue     string      `yaml:"expression,omitempty"`
+	RunTimeValue *vm.Program `yaml:"-"` // the actual compiled filter
+	// a grok can contain statics that apply if pattern is successful
+	Statics []Static `yaml:"statics,omitempty"`
+	RuntimeStatics []RuntimeStatic `yaml:"-"`
+}
+
+type DataCapture struct {
+	Name            string        `yaml:"name,omitempty"`
+	Key             string        `yaml:"key,omitempty"`
+	KeyExpression   *vm.Program   `yaml:"-"`
+	Value           string        `yaml:"value,omitempty"`
+	ValueExpression *vm.Program   `yaml:"-"`
+	TTL             string        `yaml:"ttl,omitempty"`
+	TTLVal          time.Duration `yaml:"-"`
+	MaxMapSize      int           `yaml:"size,omitempty"`
+	Strategy        string        `yaml:"strategy,omitempty"`
+}