Skip to content

fix(linter): cap checker pool and workers to 4, add panic recovery per-file #1019

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
armandthuillart wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix(linter): cap checker pool and workers to 4, add panic recovery per-file #1019

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
153 changes: 88 additions & 65 deletions internal/linter/linter.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -462,7 +462,8 @@ func RunLinterOnProgram(options RunLinterOnProgramOptions) error {
logLevel := options.LogLevel
program := options.Program
files := options.Files
workers := options.Workers
// Cap workers to prevent excessive goroutine count and GC pressure
workers := min(options.Workers, 4)
getRulesForFile := options.GetRulesForFile
onDiagnostic := options.OnDiagnostic
onInternalDiagnostic := options.OnInternalDiagnostic
Expand Down Expand Up @@ -501,38 +502,49 @@ func RunLinterOnProgram(options RunLinterOnProgramOptions) error {
ctx.TypeChecker = w.checker

for file := range w.queue {
if logLevel == utils.LogLevelDebug {
log.Print(file.FileName())
}
ctxBuilder.file = file
ctx.SourceFile = file
func() {
defer func() {
if r := recover(); r != nil {
log.Printf("panic in rule %q while processing %s: %v", ctxBuilder.ruleName, file.FileName(), r)
for k := range registeredListeners {
registeredListeners[k] = registeredListeners[k][:0]
}
}
}()

rules := getRulesForFile(file)
for _, r := range rules {
ctxBuilder.ruleName = r.Name
for kind, listener := range r.Run(ctx) {
listeners, ok := registeredListeners[kind]
if !ok {
listeners = make([]taggedListener, 0, len(rules))
if logLevel == utils.LogLevelDebug {
log.Print(file.FileName())
}
ctxBuilder.file = file
ctx.SourceFile = file

rules := getRulesForFile(file)
for _, r := range rules {
ctxBuilder.ruleName = r.Name
for kind, listener := range r.Run(ctx) {
listeners, ok := registeredListeners[kind]
if !ok {
listeners = make([]taggedListener, 0, len(rules))
}
registeredListeners[kind] = append(listeners, taggedListener{ruleName: r.Name, fn: listener})
}
registeredListeners[kind] = append(listeners, taggedListener{ruleName: r.Name, fn: listener})
}
}

runListeners := func(kind ast.Kind, node *ast.Node) {
if listeners, ok := registeredListeners[kind]; ok {
for _, listener := range listeners {
ctxBuilder.ruleName = listener.ruleName
listener.fn(node)
runListeners := func(kind ast.Kind, node *ast.Node) {
if listeners, ok := registeredListeners[kind]; ok {
for _, listener := range listeners {
ctxBuilder.ruleName = listener.ruleName
listener.fn(node)
}
}
}
}

visitLintNodes(file, runListeners)
// Instead of clearing the map, we clear the slices in-place to avoid re-allocating memory for the listeners on each file.
for k := range registeredListeners {
registeredListeners[k] = registeredListeners[k][:0]
}
visitLintNodes(file, runListeners)
// Instead of clearing the map, we clear the slices in-place to avoid re-allocating memory for the listeners on each file.
for k := range registeredListeners {
registeredListeners[k] = registeredListeners[k][:0]
}
}()
}
}

Expand All @@ -559,52 +571,63 @@ func RunLinterOnProgram(options RunLinterOnProgramOptions) error {
ctx.TypeChecker = w.checker

for file := range w.queue {
if logLevel == utils.LogLevelDebug {
log.Print(file.FileName())
}
ctxBuilder.file = file
ctx.SourceFile = file

rules := getRulesForFile(file)
timingStats := make([]RuleTimingStat, len(rules))
for ruleIdx, r := range rules {
ctxBuilder.ruleName = r.Name
start := time.Now()
listenersByKind := r.Run(ctx)
recordTiming(&timingStats[ruleIdx], time.Since(start))
for kind, listener := range listenersByKind {
listeners, ok := registeredListeners[kind]
if !ok {
listeners = make([]timedTaggedListener, 0, len(rules))
func() {
defer func() {
if r := recover(); r != nil {
log.Printf("panic in rule %q while processing %s: %v", ctxBuilder.ruleName, file.FileName(), r)
for k := range registeredListeners {
registeredListeners[k] = registeredListeners[k][:0]
}
}
}()

if logLevel == utils.LogLevelDebug {
log.Print(file.FileName())
}
ctxBuilder.file = file
ctx.SourceFile = file

rules := getRulesForFile(file)
timingStats := make([]RuleTimingStat, len(rules))
for ruleIdx, r := range rules {
ctxBuilder.ruleName = r.Name
start := time.Now()
listenersByKind := r.Run(ctx)
recordTiming(&timingStats[ruleIdx], time.Since(start))
for kind, listener := range listenersByKind {
listeners, ok := registeredListeners[kind]
if !ok {
listeners = make([]timedTaggedListener, 0, len(rules))
}
registeredListeners[kind] = append(listeners, timedTaggedListener{ruleName: r.Name, ruleIdx: ruleIdx, fn: listener})
}
registeredListeners[kind] = append(listeners, timedTaggedListener{ruleName: r.Name, ruleIdx: ruleIdx, fn: listener})
}
}

runListeners := func(kind ast.Kind, node *ast.Node) {
if listeners, ok := registeredListeners[kind]; ok {
for _, listener := range listeners {
ctxBuilder.ruleName = listener.ruleName
start := time.Now()
listener.fn(node)
recordTiming(&timingStats[listener.ruleIdx], time.Since(start))
runListeners := func(kind ast.Kind, node *ast.Node) {
if listeners, ok := registeredListeners[kind]; ok {
for _, listener := range listeners {
ctxBuilder.ruleName = listener.ruleName
start := time.Now()
listener.fn(node)
recordTiming(&timingStats[listener.ruleIdx], time.Since(start))
}
}
}
}

visitLintNodes(file, runListeners)
for idx, stat := range timingStats {
if stat.Calls == 0 {
continue
visitLintNodes(file, runListeners)
for idx, stat := range timingStats {
if stat.Calls == 0 {
continue
}
merged := localTimings[rules[idx].Name]
merged.add(stat)
localTimings[rules[idx].Name] = merged
}
merged := localTimings[rules[idx].Name]
merged.add(stat)
localTimings[rules[idx].Name] = merged
}
// Instead of clearing the map, we clear the slices in-place to avoid re-allocating memory for the listeners on each file.
for k := range registeredListeners {
registeredListeners[k] = registeredListeners[k][:0]
}
// Instead of clearing the map, we clear the slices in-place to avoid re-allocating memory for the listeners on each file.
for k := range registeredListeners {
registeredListeners[k] = registeredListeners[k][:0]
}
}()
}
}

Expand Down
2 changes: 1 addition & 1 deletion patches/0001-Adapt-project-service-for-single-run-mode.patch
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ index 37abaff97..19dac4dfc 100644
// stored in the old program's options.
createCheckerPool := func(program *compiler.Program) compiler.CheckerPool {
- checkerPool = newCheckerPool(4, program, nil)
+ checkerPool = newCheckerPool(runtime.GOMAXPROCS(0), program, p.log)
+ checkerPool = newCheckerPool(min(4, runtime.GOMAXPROCS(0)), program, p.log)
return checkerPool
}