Skip to content

Commit 5476977

Browse files
mmetcmmetc
authored
refact pkg/parser: extract RuntimeGrokPattern (#3916)
1 parent aa51a4e commit 5476977

File tree

5 files changed

+110
-79
lines changed

5 files changed

+110
-79
lines changed

‎.golangci.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -236,8 +236,8 @@ linters:
236236
- name: function-length
237237
arguments:
238238
# lower this after refactoring
239-
- 87
240-
- 198
239+
- 86
240+
- 174
241241
- name: get-return
242242
disabled: true
243243
- name: increment-decrement

‎cmd/crowdsec-cli/clilapi/context.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -306,8 +306,8 @@ func detectStaticField(grokStatics []parser.Static) []string {
306306
func detectNode(node parser.Node, parserCTX parser.UnixParserCtx) []string {
307307
ret := make([]string, 0)
308308

309-
if node.Grok.RunTimeRegexp != nil {
310-
for _, capturedField := range node.Grok.RunTimeRegexp.Names() {
309+
if node.RuntimeGrok.RunTimeRegexp != nil {
310+
for _, capturedField := range node.RuntimeGrok.RunTimeRegexp.Names() {
311311
fieldName := "evt.Parsed." + capturedField
312312
if !slices.Contains(ret, fieldName) {
313313
ret = append(ret, fieldName)
@@ -353,8 +353,8 @@ func detectSubNode(node parser.Node, parserCTX parser.UnixParserCtx) []string {
353353
ret := make([]string, 0)
354354

355355
for _, subnode := range node.LeavesNodes {
356-
if subnode.Grok.RunTimeRegexp != nil {
357-
for _, capturedField := range subnode.Grok.RunTimeRegexp.Names() {
356+
if subnode.RuntimeGrok.RunTimeRegexp != nil {
357+
for _, capturedField := range subnode.RuntimeGrok.RunTimeRegexp.Names() {
358358
fieldName := "evt.Parsed." + capturedField
359359
if !slices.Contains(ret, fieldName) {
360360
ret = append(ret, fieldName)

‎pkg/parser/grok.go‎

Lines changed: 71 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package parser
33
import (
44
"errors"
55
"fmt"
6+
"strings"
67
"time"
78

89
"github.com/expr-lang/expr"
@@ -27,13 +28,13 @@ type Static struct {
2728
// the source is a static value
2829
Value string `yaml:"value,omitempty"`
2930
// or the result of an Expression
30-
ExpValue string `yaml:"expression,omitempty"`
31+
ExpValue string `yaml:"expression,omitempty"`
3132
// or an enrichment method
3233
Method string `yaml:"method,omitempty"`
3334
}
3435

3536
type RuntimeStatic struct {
36-
Config *Static
37+
Config *Static
3738
RunTimeValue *vm.Program
3839
}
3940

@@ -84,14 +85,77 @@ type GrokPattern struct {
8485
RegexpName string `yaml:"name,omitempty"`
8586
// a proper grok pattern
8687
RegexpValue string `yaml:"pattern,omitempty"`
87-
// the runtime form of regexpname / regexpvalue
88-
RunTimeRegexp grokky.Pattern `yaml:"-"` // the actual regexp
8988
// the output of the expression is going to be the source for regexp
90-
ExpValue string `yaml:"expression,omitempty"`
91-
RunTimeValue *vm.Program `yaml:"-"` // the actual compiled filter
89+
ExpValue string `yaml:"expression,omitempty"`
9290
// a grok can contain statics that apply if pattern is successful
9391
Statics []Static `yaml:"statics,omitempty"`
94-
RuntimeStatics []RuntimeStatic `yaml:"-"`
92+
}
93+
94+
type RuntimeGrokPattern struct {
95+
Config *GrokPattern
96+
97+
RunTimeRegexp grokky.Pattern // the actual regexp
98+
RunTimeValue *vm.Program // the actual compiled filter
99+
RuntimeStatics []RuntimeStatic
100+
}
101+
102+
func (g *GrokPattern) Compile(pctx *UnixParserCtx, logger *log.Entry) (*RuntimeGrokPattern, error) {
103+
var err error
104+
105+
rg := &RuntimeGrokPattern{}
106+
/* load grok by name or compile in-place */
107+
if g.RegexpName != "" {
108+
logger.Tracef("+ Regexp Compilation %q", g.RegexpName)
109+
110+
rg.RunTimeRegexp, err = pctx.Grok.Get(g.RegexpName)
111+
if err != nil {
112+
return nil, fmt.Errorf("unable to find grok %q: %v", g.RegexpName, err)
113+
}
114+
115+
if rg.RunTimeRegexp == nil {
116+
return nil, fmt.Errorf("empty grok %q", g.RegexpName)
117+
}
118+
119+
logger.Tracef("%s regexp: %s", g.RegexpName, rg.RunTimeRegexp.String())
120+
} else if g.RegexpValue != "" {
121+
if strings.HasSuffix(g.RegexpValue, "\n") {
122+
logger.Debugf("Beware, pattern ends with \\n: %q", g.RegexpValue)
123+
}
124+
125+
rg.RunTimeRegexp, err = pctx.Grok.Compile(g.RegexpValue)
126+
if err != nil {
127+
return nil, fmt.Errorf("failed to compile grok %q: %v", g.RegexpValue, err)
128+
}
129+
130+
if rg.RunTimeRegexp == nil {
131+
// We shouldn't be here because compilation succeeded, so regexp shouldn't be nil
132+
return nil, fmt.Errorf("grok compilation failure: %s", g.RegexpValue)
133+
}
134+
135+
logger.Tracef("%s regexp: %s", g.RegexpValue, rg.RunTimeRegexp.String())
136+
}
137+
138+
// if grok source is an expression
139+
if g.ExpValue != "" {
140+
rg.RunTimeValue, err = expr.Compile(g.ExpValue,
141+
exprhelpers.GetExprOptions(map[string]any{"evt": &types.Event{}})...)
142+
if err != nil {
143+
return nil, fmt.Errorf("while compiling grok's expression: %w", err)
144+
}
145+
}
146+
147+
/* load grok statics */
148+
// compile expr statics if present
149+
for _, static := range g.Statics {
150+
compiled, err := static.Compile()
151+
if err != nil {
152+
return nil, err
153+
}
154+
155+
rg.RuntimeStatics = append(rg.RuntimeStatics, *compiled)
156+
}
157+
158+
return rg, nil
95159
}
96160

97161
type DataCapture struct {

‎pkg/parser/node.go‎

Lines changed: 17 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -57,9 +57,10 @@ type Node struct {
5757
SubGroks yaml.MapSlice `yaml:"pattern_syntax,omitempty"`
5858

5959
// Holds a grok pattern
60-
Grok GrokPattern `yaml:"grok,omitempty"`
60+
Grok GrokPattern `yaml:"grok,omitempty"`
61+
RuntimeGrok RuntimeGrokPattern `yaml:"-"`
6162
// Statics can be present in any type of node and is executed last
62-
Statics []Static `yaml:"statics,omitempty"`
63+
Statics []Static `yaml:"statics,omitempty"`
6364
RuntimeStatics []RuntimeStatic `yaml:"-"`
6465
// Stash allows to capture data from the log line and store it in an accessible cache
6566
Stash []DataCapture `yaml:"stash,omitempty"`
@@ -83,7 +84,7 @@ func (n *Node) validate(ectx EnricherCtx) error {
8384
return fmt.Errorf("non-empty filter %q was not compiled", n.Filter)
8485
}
8586

86-
if n.Grok.RunTimeRegexp != nil || n.Grok.TargetField != "" {
87+
if n.RuntimeGrok.RunTimeRegexp != nil || n.Grok.TargetField != "" {
8788
if n.Grok.TargetField == "" && n.Grok.ExpValue == "" {
8889
return errors.New("grok requires 'expression' or 'apply_on'")
8990
}
@@ -202,12 +203,12 @@ func (n *Node) processGrok(p *types.Event, cachedExprEnv map[string]any) (bool,
202203
clog := n.Logger
203204
gstr := ""
204205

205-
if n.Grok.RunTimeRegexp == nil {
206-
clog.Tracef("! No grok pattern: %p", n.Grok.RunTimeRegexp)
206+
if n.RuntimeGrok.RunTimeRegexp == nil {
207+
clog.Tracef("! No grok pattern: %p", n.RuntimeGrok.RunTimeRegexp)
207208
return true, false, nil
208209
}
209210

210-
clog.Tracef("Processing grok pattern: %s: %p", n.Grok.RegexpName, n.Grok.RunTimeRegexp)
211+
clog.Tracef("Processing grok pattern: %s: %p", n.Grok.RegexpName, n.RuntimeGrok.RunTimeRegexp)
211212
// for unparsed, parsed etc. set sensible defaults to reduce user hassle
212213
if n.Grok.TargetField != "" {
213214
// it's a hack to avoid using real reflect
@@ -219,8 +220,8 @@ func (n *Node) processGrok(p *types.Event, cachedExprEnv map[string]any) (bool,
219220
clog.Debugf("(%s) target field %q doesn't exist in %v", n.rn, n.Grok.TargetField, p.Parsed)
220221
return false, false, nil
221222
}
222-
} else if n.Grok.RunTimeValue != nil {
223-
output, err := exprhelpers.Run(n.Grok.RunTimeValue, cachedExprEnv, clog, n.Debug)
223+
} else if n.RuntimeGrok.RunTimeValue != nil {
224+
output, err := exprhelpers.Run(n.RuntimeGrok.RunTimeValue, cachedExprEnv, clog, n.Debug)
224225
if err != nil {
225226
clog.Warningf("failed to run RunTimeValue: %v", err)
226227
return false, false, nil
@@ -245,7 +246,7 @@ func (n *Node) processGrok(p *types.Event, cachedExprEnv map[string]any) (bool,
245246
groklabel = n.Grok.RegexpName
246247
}
247248

248-
grok := n.Grok.RunTimeRegexp.Parse(gstr)
249+
grok := n.RuntimeGrok.RunTimeRegexp.Parse(gstr)
249250

250251
if len(grok) == 0 {
251252
// grok failed, node failed
@@ -263,7 +264,7 @@ func (n *Node) processGrok(p *types.Event, cachedExprEnv map[string]any) (bool,
263264
p.Parsed[k] = v
264265
}
265266
// if the grok succeed, process associated statics
266-
err := n.ProcessStatics(n.Grok.RuntimeStatics, p)
267+
err := n.RuntimeGrok.ProcessStatics(p, n.EnrichFunctions, clog, n.Debug)
267268
if err != nil {
268269
clog.Errorf("(%s) Failed to process statics: %v", n.rn, err)
269270
return false, false, err
@@ -340,6 +341,7 @@ func (n *Node) processLeaves(
340341
if err != nil {
341342
n.Logger.Tracef("\tNode (%s) failed: %v", child.rn, err)
342343
n.Logger.Debugf("Event leaving node: ko")
344+
343345
return false, err
344346
}
345347

@@ -395,7 +397,7 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx, expressionEnv map[stri
395397
}
396398

397399
// Process the stash (data collection) if: a grok was present and succeeded, or if there is no grok
398-
if nodeHasOKGrok || n.Grok.RunTimeRegexp == nil {
400+
if nodeHasOKGrok || n.RuntimeGrok.RunTimeRegexp == nil {
399401
if err := n.processStash(p, cachedExprEnv, clog); err != nil {
400402
return false, err
401403
}
@@ -435,7 +437,7 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx, expressionEnv map[stri
435437
if len(n.Statics) > 0 && (isWhitelisted || !n.ContainsWLs()) {
436438
clog.Debugf("+ Processing %d statics", len(n.Statics))
437439
// if all else is good in whitelist, process node's statics
438-
err := n.ProcessStatics(n.RuntimeStatics, p)
440+
err := n.ProcessStatics(p)
439441
if err != nil {
440442
clog.Errorf("Failed to process statics: %v", err)
441443
return false, err
@@ -523,61 +525,13 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error {
523525
}
524526
}
525527

526-
/* load grok by name or compile in-place */
527-
if n.Grok.RegexpName != "" {
528-
n.Logger.Tracef("+ Regexp Compilation %q", n.Grok.RegexpName)
529-
530-
n.Grok.RunTimeRegexp, err = pctx.Grok.Get(n.Grok.RegexpName)
531-
if err != nil {
532-
return fmt.Errorf("unable to find grok %q: %v", n.Grok.RegexpName, err)
533-
}
534-
535-
if n.Grok.RunTimeRegexp == nil {
536-
return fmt.Errorf("empty grok %q", n.Grok.RegexpName)
537-
}
538-
539-
n.Logger.Tracef("%s regexp: %s", n.Grok.RegexpName, n.Grok.RunTimeRegexp.String())
540-
541-
valid = true
542-
} else if n.Grok.RegexpValue != "" {
543-
if strings.HasSuffix(n.Grok.RegexpValue, "\n") {
544-
n.Logger.Debugf("Beware, pattern ends with \\n: %q", n.Grok.RegexpValue)
545-
}
546-
547-
n.Grok.RunTimeRegexp, err = pctx.Grok.Compile(n.Grok.RegexpValue)
548-
if err != nil {
549-
return fmt.Errorf("failed to compile grok %q: %v", n.Grok.RegexpValue, err)
550-
}
551-
552-
if n.Grok.RunTimeRegexp == nil {
553-
// We shouldn't be here because compilation succeeded, so regexp shouldn't be nil
554-
return fmt.Errorf("grok compilation failure: %s", n.Grok.RegexpValue)
555-
}
556-
557-
n.Logger.Tracef("%s regexp: %s", n.Grok.RegexpValue, n.Grok.RunTimeRegexp.String())
558-
559-
valid = true
560-
}
561-
562-
// if grok source is an expression
563-
if n.Grok.ExpValue != "" {
564-
n.Grok.RunTimeValue, err = expr.Compile(n.Grok.ExpValue,
565-
exprhelpers.GetExprOptions(map[string]any{"evt": &types.Event{}})...)
566-
if err != nil {
567-
return fmt.Errorf("while compiling grok's expression: %w", err)
568-
}
569-
}
570-
571-
/* load grok statics */
572-
// compile expr statics if present
573-
for _, static := range n.Grok.Statics {
574-
compiled, err := static.Compile()
528+
if n.Grok.RegexpName != "" || n.Grok.RegexpValue != "" || n.Grok.ExpValue != "" {
529+
rg, err := n.Grok.Compile(pctx, n.Logger)
575530
if err != nil {
576531
return err
577532
}
578533

579-
n.Grok.RuntimeStatics = append(n.Grok.RuntimeStatics, *compiled)
580-
534+
n.RuntimeGrok = *rg
581535
valid = true
582536
}
583537

‎pkg/parser/runtime.go‎

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -212,10 +212,20 @@ func (rs *RuntimeStatic) Apply(event *types.Event, enrichFunctions EnricherCtx,
212212
return nil
213213
}
214214

215-
func (n *Node) ProcessStatics(statics []RuntimeStatic, event *types.Event) error {
216-
for _, rs := range statics {
215+
func (n *Node) ProcessStatics(event *types.Event) error {
216+
for _, rs := range n.RuntimeStatics {
217217
if err := rs.Apply(event, n.EnrichFunctions, n.Logger, n.Debug); err != nil {
218-
return fmt.Errorf("applying static %s: %w", rs.Config.targetExpr(), err)
218+
return fmt.Errorf("applying %s: %w", rs.Config.targetExpr(), err)
219+
}
220+
}
221+
222+
return nil
223+
}
224+
225+
func (rg *RuntimeGrokPattern) ProcessStatics(event *types.Event, ectx EnricherCtx, logger *log.Entry, debug bool) error {
226+
for _, rs := range rg.RuntimeStatics {
227+
if err := rs.Apply(event, ectx, logger, debug); err != nil {
228+
return fmt.Errorf("applying %s: %w", rs.Config.targetExpr(), err)
219229
}
220230
}
221231

@@ -301,6 +311,7 @@ func Parse(ctx UnixParserCtx, xp types.Event, nodes []Node) (types.Event, error)
301311
if event.Stage != stage {
302312
log.Debugf("Event not parsed, expected stage '%s' got '%s', abort", stage, event.Stage)
303313
event.Process = false
314+
304315
return event, nil
305316
}
306317

@@ -340,6 +351,7 @@ func Parse(ctx UnixParserCtx, xp types.Event, nodes []Node) (types.Event, error)
340351

341352
// ensure the stage map exists
342353
StageParseMutex.Lock()
354+
343355
stageMap, ok := StageParseCache[stage]
344356
if !ok {
345357
stageMap = make(map[string][]dumps.ParserResult)
@@ -381,6 +393,7 @@ func Parse(ctx UnixParserCtx, xp types.Event, nodes []Node) (types.Event, error)
381393
if !isStageOK {
382394
log.Debugf("Log didn't finish stage %s", event.Stage)
383395
event.Process = false
396+
384397
return event, nil
385398
}
386399
}

0 commit comments

Comments
 (0)