Skip to content

Commit 0222117

Browse files
kaskol10kaskol10
committed
Improve Karpenter log analyzer for reconciler and admission webhook errors
- Add error categorization for admission webhook, reconciler, and eviction errors - Improve summary generation to use name/namespace from error itself (not just Pod fields) - Always analyze main Error field, even when errorCauses is empty - Enhance AI prompt with specific guidance for different error types - For reconciler errors, emphasize analyzing Error field which contains actual failure - Add Strimzi drain cleaner webhook detection and explanation - Improve context building to prioritize error-level name/namespace over Pod fields - Update recommendations to include guidance for admission webhook and reconciler errors
1 parent 9a1bd24 commit 0222117

1 file changed

Lines changed: 215 additions & 23 deletions

File tree

‎internal/api/karpenter_log.go‎

Lines changed: 215 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,38 @@ func parseKarpenterLog(logStr string) (*KarpenterLogError, error) {
7373
func categorizeErrorCause(errorMsg string) (category, explanation, severity string) {
7474
errorLower := strings.ToLower(errorMsg)
7575

76+
// Admission webhook errors
77+
if strings.Contains(errorLower, "admission webhook") || strings.Contains(errorLower, "denied the request") {
78+
webhookName := ""
79+
if strings.Contains(errorMsg, "admission webhook") {
80+
parts := strings.Split(errorMsg, "admission webhook")
81+
if len(parts) > 1 {
82+
webhookPart := strings.TrimSpace(parts[1])
83+
if strings.Contains(webhookPart, "denied") {
84+
webhookName = strings.Split(webhookPart, "denied")[0]
85+
webhookName = strings.Trim(webhookName, `"`)
86+
}
87+
}
88+
}
89+
return "Admission Webhook",
90+
fmt.Sprintf("An admission webhook (%s) denied the request. This is typically a policy enforcement (e.g., Strimzi drain cleaner, Pod Security Standards). Check webhook configuration and pod spec compliance.", webhookName),
91+
"critical"
92+
}
93+
94+
// Reconciler errors
95+
if strings.Contains(errorLower, "reconciler error") {
96+
return "Reconciler Error",
97+
"A controller reconciler encountered an error. This could be due to resource conflicts, webhook denials, or controller logic issues. Check the specific error message and controller logs.",
98+
"critical"
99+
}
100+
101+
// Eviction errors
102+
if strings.Contains(errorLower, "eviction") || strings.Contains(errorLower, "drain") {
103+
return "Eviction/Drain Error",
104+
"Pod eviction or node draining failed. This could be due to PodDisruptionBudgets, admission webhooks, or other protection mechanisms preventing the operation.",
105+
"warning"
106+
}
107+
76108
// Label/typo errors
77109
if strings.Contains(errorLower, "does not have known values") || strings.Contains(errorLower, "typo") {
78110
return "Label Error",
@@ -125,14 +157,102 @@ func (s *Server) analyzeKarpenterLogInternal(ctx context.Context, logStr string)
125157
}, nil
126158
}
127159

128-
// Build summary
129-
summary := fmt.Sprintf("Pod '%s' in namespace '%s' could not be scheduled by NodePool '%s'",
130-
parsedError.Pod.Name, parsedError.Pod.Namespace, parsedError.NodePool.Name)
160+
// Build summary - handle cases where pod/namespace might be missing
161+
// First, determine the error type from the message or error causes
162+
errorType := "scheduling"
163+
if parsedError.Message != "" {
164+
msgLower := strings.ToLower(parsedError.Message)
165+
if strings.Contains(msgLower, "reconciler") {
166+
errorType = "reconciler"
167+
} else if strings.Contains(msgLower, "eviction") || strings.Contains(msgLower, "drain") {
168+
errorType = "eviction"
169+
} else if strings.Contains(msgLower, "admission") || strings.Contains(msgLower, "webhook") {
170+
errorType = "admission webhook"
171+
}
172+
}
131173

132-
// Analyze error causes
133-
errorCauses := make([]ErrorCauseDetail, 0, len(parsedError.ErrorCauses))
174+
// Check error causes for additional context
175+
if len(parsedError.ErrorCauses) > 0 {
176+
for _, cause := range parsedError.ErrorCauses {
177+
causeLower := strings.ToLower(cause.Error)
178+
if strings.Contains(causeLower, "admission webhook") || strings.Contains(causeLower, "denied") {
179+
errorType = "admission webhook"
180+
break
181+
} else if strings.Contains(causeLower, "reconciler") {
182+
errorType = "reconciler"
183+
break
184+
} else if strings.Contains(causeLower, "eviction") || strings.Contains(causeLower, "drain") {
185+
errorType = "eviction"
186+
break
187+
}
188+
}
189+
}
190+
191+
var summary string
192+
if parsedError.Name != "" && parsedError.Namespace != "" {
193+
// Use name/namespace from the error itself (not Pod.Name/Pod.Namespace)
194+
if errorType == "reconciler" {
195+
summary = fmt.Sprintf("Reconciler error for '%s' in namespace '%s'", parsedError.Name, parsedError.Namespace)
196+
} else if errorType == "admission webhook" {
197+
summary = fmt.Sprintf("Admission webhook denied request for '%s' in namespace '%s'", parsedError.Name, parsedError.Namespace)
198+
} else if errorType == "eviction" {
199+
summary = fmt.Sprintf("Eviction error for '%s' in namespace '%s'", parsedError.Name, parsedError.Namespace)
200+
} else if parsedError.Pod.Name != "" && parsedError.Pod.Namespace != "" {
201+
if parsedError.NodePool.Name != "" {
202+
summary = fmt.Sprintf("Pod '%s' in namespace '%s' could not be scheduled by NodePool '%s'",
203+
parsedError.Pod.Name, parsedError.Pod.Namespace, parsedError.NodePool.Name)
204+
} else {
205+
summary = fmt.Sprintf("Pod '%s' in namespace '%s' could not be scheduled",
206+
parsedError.Pod.Name, parsedError.Pod.Namespace)
207+
}
208+
} else {
209+
summary = fmt.Sprintf("Karpenter %s error for '%s' in namespace '%s'", errorType, parsedError.Name, parsedError.Namespace)
210+
}
211+
} else if parsedError.Pod.Name != "" && parsedError.Pod.Namespace != "" {
212+
if parsedError.NodePool.Name != "" {
213+
summary = fmt.Sprintf("Pod '%s' in namespace '%s' could not be scheduled by NodePool '%s'",
214+
parsedError.Pod.Name, parsedError.Pod.Namespace, parsedError.NodePool.Name)
215+
} else {
216+
summary = fmt.Sprintf("Pod '%s' in namespace '%s' could not be scheduled",
217+
parsedError.Pod.Name, parsedError.Pod.Namespace)
218+
}
219+
} else if parsedError.NodePool.Name != "" {
220+
summary = fmt.Sprintf("Karpenter %s error for NodePool '%s'", errorType, parsedError.NodePool.Name)
221+
} else if parsedError.Message != "" {
222+
summary = fmt.Sprintf("Karpenter %s error: %s", errorType, parsedError.Message)
223+
} else {
224+
summary = fmt.Sprintf("Karpenter %s error detected", errorType)
225+
}
226+
227+
// Analyze error causes - include main error field if errorCauses is empty
228+
errorCauses := make([]ErrorCauseDetail, 0)
134229
seenErrors := make(map[string]bool)
135230

231+
// First, analyze the main error field if it's not already in errorCauses
232+
// This is important for reconciler errors where the main error contains the actual issue
233+
if parsedError.Error != "" {
234+
// Check if this error is already in errorCauses
235+
foundInCauses := false
236+
for _, cause := range parsedError.ErrorCauses {
237+
if cause.Error == parsedError.Error {
238+
foundInCauses = true
239+
break
240+
}
241+
}
242+
// If not found in errorCauses, analyze it (especially important when errorCauses is empty)
243+
if !foundInCauses {
244+
category, explanation, severity := categorizeErrorCause(parsedError.Error)
245+
errorCauses = append(errorCauses, ErrorCauseDetail{
246+
Error: parsedError.Error,
247+
Category: category,
248+
Explanation: explanation,
249+
Severity: severity,
250+
})
251+
seenErrors[parsedError.Error] = true
252+
}
253+
}
254+
255+
// Then analyze errorCauses
136256
for _, cause := range parsedError.ErrorCauses {
137257
// Deduplicate similar errors
138258
if seenErrors[cause.Error] {
@@ -206,42 +326,114 @@ func (s *Server) generateAIExplanation(ctx context.Context, parsedError *Karpent
206326
return "", fmt.Errorf("ollama client not available")
207327
}
208328

209-
// Build prompt
329+
// Build prompt with available information
210330
causesText := ""
211331
for i, cause := range errorCauses {
212332
causesText += fmt.Sprintf("%d. [%s] %s\n Explanation: %s\n",
213333
i+1, cause.Severity, cause.Error, cause.Explanation)
214334
}
215335

216-
taintsText := ""
336+
// Build context information dynamically based on what's available
337+
contextParts := []string{}
338+
339+
// Use name/namespace from the error itself first (for reconciler errors, eviction errors, etc.)
340+
if parsedError.Name != "" && parsedError.Namespace != "" {
341+
contextParts = append(contextParts, fmt.Sprintf("- Resource Name: %s", parsedError.Name))
342+
contextParts = append(contextParts, fmt.Sprintf("- Resource Namespace: %s", parsedError.Namespace))
343+
} else if parsedError.Pod.Name != "" && parsedError.Pod.Namespace != "" {
344+
contextParts = append(contextParts, fmt.Sprintf("- Pod Name: %s", parsedError.Pod.Name))
345+
contextParts = append(contextParts, fmt.Sprintf("- Pod Namespace: %s", parsedError.Pod.Namespace))
346+
} else if parsedError.Pod.Name != "" {
347+
contextParts = append(contextParts, fmt.Sprintf("- Pod Name: %s", parsedError.Pod.Name))
348+
} else if parsedError.Pod.Namespace != "" {
349+
contextParts = append(contextParts, fmt.Sprintf("- Pod Namespace: %s", parsedError.Pod.Namespace))
350+
}
351+
352+
if parsedError.NodePool.Name != "" {
353+
contextParts = append(contextParts, fmt.Sprintf("- NodePool: %s", parsedError.NodePool.Name))
354+
}
355+
356+
if parsedError.Message != "" {
357+
contextParts = append(contextParts, fmt.Sprintf("- Error Message: %s", parsedError.Message))
358+
}
359+
360+
if parsedError.Controller != "" {
361+
contextParts = append(contextParts, fmt.Sprintf("- Controller: %s", parsedError.Controller))
362+
}
363+
217364
if len(parsedError.Taints) > 0 {
218-
taintsText = strings.Join(parsedError.Taints, ", ")
365+
taintsText := strings.Join(parsedError.Taints, ", ")
366+
contextParts = append(contextParts, fmt.Sprintf("- Taints: %s", taintsText))
219367
}
220368

221-
prompt := fmt.Sprintf(`You are a Kubernetes and Karpenter expert. Analyze this Karpenter scheduling error and provide a clear, actionable explanation.
369+
// Add the main error field (critical for reconciler errors where the actual error is here)
370+
if parsedError.Error != "" {
371+
contextParts = append(contextParts, fmt.Sprintf("- Error: %s", parsedError.Error))
372+
}
222373

223-
Pod Details:
224-
- Name: %s
225-
- Namespace: %s
226-
- NodePool: %s
374+
contextInfo := strings.Join(contextParts, "\n")
375+
if contextInfo == "" {
376+
contextInfo = "No specific pod, namespace, or NodePool information available in the log."
377+
}
227378

228-
Taints on NodePool:
379+
// Determine error type for better prompt context
380+
errorType := "scheduling"
381+
if parsedError.Message != "" {
382+
msgLower := strings.ToLower(parsedError.Message)
383+
if strings.Contains(msgLower, "reconciler") {
384+
errorType = "reconciler"
385+
} else if strings.Contains(msgLower, "eviction") || strings.Contains(msgLower, "drain") {
386+
errorType = "eviction"
387+
} else if strings.Contains(msgLower, "admission") || strings.Contains(msgLower, "webhook") {
388+
errorType = "admission webhook"
389+
}
390+
}
391+
392+
// Check error causes for error type hints
393+
if len(errorCauses) > 0 {
394+
for _, cause := range errorCauses {
395+
causeLower := strings.ToLower(cause.Error)
396+
if strings.Contains(causeLower, "admission webhook") || strings.Contains(causeLower, "denied") {
397+
errorType = "admission webhook"
398+
break
399+
} else if strings.Contains(causeLower, "reconciler") {
400+
errorType = "reconciler"
401+
break
402+
} else if strings.Contains(causeLower, "eviction") || strings.Contains(causeLower, "drain") {
403+
errorType = "eviction"
404+
break
405+
}
406+
}
407+
}
408+
409+
prompt := fmt.Sprintf(`You are a Kubernetes and Karpenter expert. Analyze this Karpenter error and provide a clear, actionable explanation.
410+
411+
Error Type: %s
412+
413+
Available Context:
229414
%s
230415
231416
Error Causes:
232417
%s
233418
234419
Provide a concise explanation (2-3 sentences) that:
235-
1. Summarizes why the pod cannot be scheduled
236-
2. Identifies the primary issue(s)
237-
3. Suggests actionable steps to resolve the problem
238-
239-
Be specific and technical, but clear. Focus on the most critical issues first.`,
240-
parsedError.Pod.Name,
241-
parsedError.Pod.Namespace,
242-
parsedError.NodePool.Name,
243-
taintsText,
420+
1. Identifies what type of error this is (%s error) and what it means
421+
2. Explains the root cause based on the error message and error causes
422+
3. Suggests specific, actionable steps to resolve the problem
423+
424+
Be specific and technical, but clear. Focus on the most critical issues first.
425+
- For reconciler errors: The "Error" field contains the actual failure reason (often an admission webhook denial). Explain what the controller (shown in "Controller" field) was trying to do, identify the specific webhook or issue that blocked it, and why. For example, if it's a Strimzi drain cleaner webhook, explain that it's preventing pod eviction because the Strimzi operator will handle rolling the pod.
426+
- For admission webhook errors: Identify which webhook denied the request (extract from the error message), explain why it denied (e.g., Strimzi drain cleaner prevents manual eviction, Pod Security Standards violation), and what action is needed.
427+
- For eviction errors: Explain what prevented the eviction (PodDisruptionBudgets, admission webhooks, etc.) and how to resolve it.
428+
- For scheduling errors: Focus on resource constraints, taints, labels, or NodePool limits.
429+
430+
IMPORTANT: For reconciler errors, the "Error" field is the key - it contains the actual failure (often an admission webhook denial). Always analyze this field even if error causes are empty.
431+
432+
If pod/namespace information is missing, analyze the error message, error field, and error causes to provide useful guidance.`,
433+
errorType,
434+
contextInfo,
244435
causesText,
436+
errorType,
245437
)
246438

247439
// Use timeout for AI request

0 commit comments

Comments
 (0)