|
8 | 8 | "testing" |
9 | 9 | "time" |
10 | 10 |
|
| 11 | + "github.com/prometheus/client_golang/prometheus" |
| 12 | + dto "github.com/prometheus/client_model/go" |
11 | 13 | "github.com/stretchr/testify/require" |
12 | 14 | v1 "k8s.io/api/core/v1" |
13 | 15 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" |
@@ -55,6 +57,22 @@ func TestSolveWithResultReturnsTerminalResultWhenNoTasksToAllocate(t *testing.T) |
55 | 57 | require.False(t, result.ReducedBudget()) |
56 | 58 | } |
57 | 59 |
|
| 60 | +func TestSolveWithResultRecordsNoSearchMetricAsNotAttempted(t *testing.T) { |
| 61 | + labels := map[string]string{ |
| 62 | + "action": "reclaim", |
| 63 | + "result": string(SearchResultNotAttempted), |
| 64 | + "reduced_budget": "false", |
| 65 | + } |
| 66 | + before := scenarioSearchCounterValue(t, "scenario_search_jobs_total", labels) |
| 67 | + solver := NewJobsSolver(nil, nil, nil, framework.Reclaim, nil) |
| 68 | + pendingJob := podgroup_info.NewPodGroupInfo("pending-job") |
| 69 | + |
| 70 | + _, _, _, result := solver.SolveWithResult(&framework.Session{}, pendingJob) |
| 71 | + |
| 72 | + require.Equal(t, SearchResultGeneratorsExhausted, result.Reason()) |
| 73 | + require.Equal(t, before+1, scenarioSearchCounterValue(t, "scenario_search_jobs_total", labels)) |
| 74 | +} |
| 75 | + |
58 | 76 | func TestSolveWithResultReturnsNoGeneratorWhenGeneratorFuncIsNil(t *testing.T) { |
59 | 77 | ssn, pendingJob := newJobSolverResultTestSession(t, 1) |
60 | 78 | solver := NewJobsSolver(nil, nil, nil, framework.Reclaim, nil) |
@@ -159,6 +177,65 @@ func TestSolveWithResultReportsDeadlineWhenBudgetExhaustsDuringScenarioSearch(t |
159 | 177 | require.Equal(t, SearchResultDeadlineExhausted, result.Reason()) |
160 | 178 | } |
161 | 179 |
|
| 180 | +func TestSolveWithResultRecordsGeneratorExhaustedMetricAfterGeneratorAttempt(t *testing.T) { |
| 181 | + labels := map[string]string{ |
| 182 | + "action": "reclaim", |
| 183 | + "result": string(SearchResultGeneratorsExhausted), |
| 184 | + "reduced_budget": "false", |
| 185 | + } |
| 186 | + before := scenarioSearchCounterValue(t, "scenario_search_jobs_total", labels) |
| 187 | + ssn, pendingJob := newJobSolverResultTestSession(t, 1) |
| 188 | + ssn.AddScenarioGenerator("empty", portfolioTestFactory(&portfolioTestGenerator{name: "empty"}), framework.Reclaim) |
| 189 | + solver := NewJobsSolver( |
| 190 | + nil, |
| 191 | + nil, |
| 192 | + func() *utils.JobsOrderByQueues { |
| 193 | + return utils.GetVictimsQueue(ssn, nil) |
| 194 | + }, |
| 195 | + framework.Reclaim, |
| 196 | + nil, |
| 197 | + ) |
| 198 | + |
| 199 | + _, _, _, result := solver.SolveWithResult(ssn, pendingJob) |
| 200 | + |
| 201 | + require.Equal(t, SearchResultGeneratorsExhausted, result.Reason()) |
| 202 | + require.Equal(t, before+1, scenarioSearchCounterValue(t, "scenario_search_jobs_total", labels)) |
| 203 | +} |
| 204 | + |
| 205 | +func TestSolveWithResultRecordsUnsolvedScenarioDurationAfterSimulation(t *testing.T) { |
| 206 | + generatorName := "test-unsolved-duration" |
| 207 | + labels := map[string]string{ |
| 208 | + "action": "reclaim", |
| 209 | + "generator": generatorName, |
| 210 | + "result": scenarioSearchResultUnsolved, |
| 211 | + } |
| 212 | + before := scenarioSearchHistogramCount(t, "scenario_search_duration_seconds", labels) |
| 213 | + ssn, pendingJob := newJobSolverResultTestSession(t, 1) |
| 214 | + ssn.ClusterInfo.Nodes = map[string]*node_info.NodeInfo{"node-1": {}} |
| 215 | + scenarioToSolve := scenario.NewByNodeScenario( |
| 216 | + ssn, pendingJob, |
| 217 | + podgroup_info.GetTasksToAllocate(pendingJob, ssn.SubGroupOrderFn, ssn.TaskOrderFn, false), |
| 218 | + nil, nil, |
| 219 | + ) |
| 220 | + ssn.AddScenarioGenerator(generatorName, portfolioTestFactory(&portfolioTestGenerator{ |
| 221 | + name: generatorName, |
| 222 | + scenarios: []api.ScenarioInfo{scenarioToSolve}, |
| 223 | + }), framework.Reclaim) |
| 224 | + solver := NewJobsSolver( |
| 225 | + nil, |
| 226 | + nil, |
| 227 | + func() *utils.JobsOrderByQueues { |
| 228 | + return utils.GetVictimsQueue(ssn, nil) |
| 229 | + }, |
| 230 | + framework.Reclaim, |
| 231 | + nil, |
| 232 | + ) |
| 233 | + |
| 234 | + solver.SolveWithResult(ssn, pendingJob) |
| 235 | + |
| 236 | + require.Equal(t, before+1, scenarioSearchHistogramCount(t, "scenario_search_duration_seconds", labels)) |
| 237 | +} |
| 238 | + |
162 | 239 | func TestSolveWithResultRunsCompletePartialSearchForOneGeneratorBeforeNext(t *testing.T) { |
163 | 240 | ssn := newGeneratorTestSession(t, map[string]int{ |
164 | 241 | "node-1": 1, |
@@ -276,3 +353,54 @@ func newJobSolverResultTestSession(t *testing.T, tasksCount int) (*framework.Ses |
276 | 353 | }, |
277 | 354 | }, pendingJob |
278 | 355 | } |
| 356 | + |
| 357 | +func scenarioSearchCounterValue(t *testing.T, metricName string, labels map[string]string) float64 { |
| 358 | + t.Helper() |
| 359 | + |
| 360 | + metric := scenarioSearchMetric(t, metricName, labels) |
| 361 | + if metric == nil || metric.GetCounter() == nil { |
| 362 | + return 0 |
| 363 | + } |
| 364 | + return metric.GetCounter().GetValue() |
| 365 | +} |
| 366 | + |
| 367 | +func scenarioSearchHistogramCount(t *testing.T, metricName string, labels map[string]string) uint64 { |
| 368 | + t.Helper() |
| 369 | + |
| 370 | + metric := scenarioSearchMetric(t, metricName, labels) |
| 371 | + if metric == nil || metric.GetHistogram() == nil { |
| 372 | + return 0 |
| 373 | + } |
| 374 | + return metric.GetHistogram().GetSampleCount() |
| 375 | +} |
| 376 | + |
| 377 | +func scenarioSearchMetric(t *testing.T, metricName string, labels map[string]string) *dto.Metric { |
| 378 | + t.Helper() |
| 379 | + |
| 380 | + families, err := prometheus.DefaultGatherer.Gather() |
| 381 | + require.NoError(t, err) |
| 382 | + for _, family := range families { |
| 383 | + if family.GetName() != metricName { |
| 384 | + continue |
| 385 | + } |
| 386 | + for _, metric := range family.GetMetric() { |
| 387 | + if scenarioSearchMetricHasLabels(metric, labels) { |
| 388 | + return metric |
| 389 | + } |
| 390 | + } |
| 391 | + } |
| 392 | + return nil |
| 393 | +} |
| 394 | + |
| 395 | +func scenarioSearchMetricHasLabels(metric *dto.Metric, labels map[string]string) bool { |
| 396 | + if len(metric.GetLabel()) != len(labels) { |
| 397 | + return false |
| 398 | + } |
| 399 | + for _, label := range metric.GetLabel() { |
| 400 | + expectedValue, found := labels[label.GetName()] |
| 401 | + if !found || expectedValue != label.GetValue() { |
| 402 | + return false |
| 403 | + } |
| 404 | + } |
| 405 | + return true |
| 406 | +} |
0 commit comments