|
8 | 8 | "testing" |
9 | 9 | "time" |
10 | 10 |
|
| 11 | + "github.com/prometheus/client_golang/prometheus" |
| 12 | + dto "github.com/prometheus/client_model/go" |
11 | 13 | "github.com/stretchr/testify/require" |
12 | 14 | v1 "k8s.io/api/core/v1" |
13 | 15 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" |
@@ -55,6 +57,23 @@ func TestSolveWithResultReturnsTerminalResultWhenNoTasksToAllocate(t *testing.T) |
55 | 57 | require.False(t, result.EnteredSearch()) |
56 | 58 | } |
57 | 59 |
|
| 60 | +func TestSolveWithResultRecordsNoSearchMetricAsNotAttempted(t *testing.T) { |
| 61 | + labels := map[string]string{ |
| 62 | + "action": "reclaim", |
| 63 | + "result": string(SearchResultNotAttempted), |
| 64 | + "reduced_budget": "false", |
| 65 | + } |
| 66 | + before := scenarioSearchCounterValue(t, "scenario_search_jobs_total", labels) |
| 67 | + solver := NewJobsSolver(nil, nil, nil, framework.Reclaim, nil) |
| 68 | + pendingJob := podgroup_info.NewPodGroupInfo("pending-job") |
| 69 | + |
| 70 | + _, _, _, result := solver.SolveWithResult(&framework.Session{}, pendingJob) |
| 71 | + |
| 72 | + require.Equal(t, SearchResultGeneratorsExhausted, result.Reason()) |
| 73 | + require.False(t, result.EnteredSearch()) |
| 74 | + require.Equal(t, before+1, scenarioSearchCounterValue(t, "scenario_search_jobs_total", labels)) |
| 75 | +} |
| 76 | + |
58 | 77 | func TestSolveWithResultReturnsNoGeneratorWhenGeneratorFuncIsNil(t *testing.T) { |
59 | 78 | ssn, pendingJob := newJobSolverResultTestSession(t, 1) |
60 | 79 | solver := NewJobsSolver(nil, nil, nil, framework.Reclaim, nil) |
@@ -91,6 +110,66 @@ func TestSolveWithResultReturnsNoGeneratorWhenGeneratorReturnsNil(t *testing.T) |
91 | 110 | require.False(t, result.EnteredSearch()) |
92 | 111 | } |
93 | 112 |
|
| 113 | +func TestSolveWithResultRecordsGeneratorExhaustedMetricAfterGeneratorAttempt(t *testing.T) { |
| 114 | + labels := map[string]string{ |
| 115 | + "action": "reclaim", |
| 116 | + "result": string(SearchResultGeneratorsExhausted), |
| 117 | + "reduced_budget": "false", |
| 118 | + } |
| 119 | + before := scenarioSearchCounterValue(t, "scenario_search_jobs_total", labels) |
| 120 | + ssn, pendingJob := newJobSolverResultTestSession(t, 1) |
| 121 | + ssn.AddScenarioGenerator("empty", portfolioTestFactory(&portfolioTestGenerator{name: "empty"}), framework.Reclaim) |
| 122 | + solver := NewJobsSolver( |
| 123 | + nil, |
| 124 | + nil, |
| 125 | + func() *utils.JobsOrderByQueues { |
| 126 | + return utils.GetVictimsQueue(ssn, nil) |
| 127 | + }, |
| 128 | + framework.Reclaim, |
| 129 | + nil, |
| 130 | + ) |
| 131 | + |
| 132 | + _, _, _, result := solver.SolveWithResult(ssn, pendingJob) |
| 133 | + |
| 134 | + require.Equal(t, SearchResultGeneratorsExhausted, result.Reason()) |
| 135 | + require.False(t, result.EnteredSearch()) |
| 136 | + require.Equal(t, before+1, scenarioSearchCounterValue(t, "scenario_search_jobs_total", labels)) |
| 137 | +} |
| 138 | + |
| 139 | +func TestSolveWithResultRecordsUnsolvedScenarioDurationAfterSimulation(t *testing.T) { |
| 140 | + generatorName := "test-unsolved-duration" |
| 141 | + labels := map[string]string{ |
| 142 | + "action": "reclaim", |
| 143 | + "generator": generatorName, |
| 144 | + "result": scenarioSearchResultUnsolved, |
| 145 | + } |
| 146 | + before := scenarioSearchHistogramCount(t, "scenario_search_duration_seconds", labels) |
| 147 | + ssn, pendingJob := newJobSolverResultTestSession(t, 1) |
| 148 | + ssn.ClusterInfo.Nodes = map[string]*node_info.NodeInfo{"node-1": {}} |
| 149 | + scenarioToSolve := scenario.NewByNodeScenario( |
| 150 | + ssn, pendingJob, |
| 151 | + podgroup_info.GetTasksToAllocate(pendingJob, ssn.SubGroupOrderFn, ssn.TaskOrderFn, false), |
| 152 | + nil, nil, |
| 153 | + ) |
| 154 | + ssn.AddScenarioGenerator(generatorName, portfolioTestFactory(&portfolioTestGenerator{ |
| 155 | + name: generatorName, |
| 156 | + scenarios: []api.ScenarioInfo{scenarioToSolve}, |
| 157 | + }), framework.Reclaim) |
| 158 | + solver := NewJobsSolver( |
| 159 | + nil, |
| 160 | + nil, |
| 161 | + func() *utils.JobsOrderByQueues { |
| 162 | + return utils.GetVictimsQueue(ssn, nil) |
| 163 | + }, |
| 164 | + framework.Reclaim, |
| 165 | + nil, |
| 166 | + ) |
| 167 | + |
| 168 | + solver.SolveWithResult(ssn, pendingJob) |
| 169 | + |
| 170 | + require.Equal(t, before+1, scenarioSearchHistogramCount(t, "scenario_search_duration_seconds", labels)) |
| 171 | +} |
| 172 | + |
94 | 173 | func TestSearchMaxSolvableKSkipsSingleTaskFullProbe(t *testing.T) { |
95 | 174 | ssn, pendingJob := newJobSolverResultTestSession(t, 1) |
96 | 175 | actionBudget := newUnlimitedActionSearchBudget(framework.Reclaim) |
@@ -277,3 +356,54 @@ func newJobSolverResultTestSession(t *testing.T, tasksCount int) (*framework.Ses |
277 | 356 | }, |
278 | 357 | }, pendingJob |
279 | 358 | } |
| 359 | + |
| 360 | +func scenarioSearchCounterValue(t *testing.T, metricName string, labels map[string]string) float64 { |
| 361 | + t.Helper() |
| 362 | + |
| 363 | + metric := scenarioSearchMetric(t, metricName, labels) |
| 364 | + if metric == nil || metric.GetCounter() == nil { |
| 365 | + return 0 |
| 366 | + } |
| 367 | + return metric.GetCounter().GetValue() |
| 368 | +} |
| 369 | + |
| 370 | +func scenarioSearchHistogramCount(t *testing.T, metricName string, labels map[string]string) uint64 { |
| 371 | + t.Helper() |
| 372 | + |
| 373 | + metric := scenarioSearchMetric(t, metricName, labels) |
| 374 | + if metric == nil || metric.GetHistogram() == nil { |
| 375 | + return 0 |
| 376 | + } |
| 377 | + return metric.GetHistogram().GetSampleCount() |
| 378 | +} |
| 379 | + |
| 380 | +func scenarioSearchMetric(t *testing.T, metricName string, labels map[string]string) *dto.Metric { |
| 381 | + t.Helper() |
| 382 | + |
| 383 | + families, err := prometheus.DefaultGatherer.Gather() |
| 384 | + require.NoError(t, err) |
| 385 | + for _, family := range families { |
| 386 | + if family.GetName() != metricName { |
| 387 | + continue |
| 388 | + } |
| 389 | + for _, metric := range family.GetMetric() { |
| 390 | + if scenarioSearchMetricHasLabels(metric, labels) { |
| 391 | + return metric |
| 392 | + } |
| 393 | + } |
| 394 | + } |
| 395 | + return nil |
| 396 | +} |
| 397 | + |
| 398 | +func scenarioSearchMetricHasLabels(metric *dto.Metric, labels map[string]string) bool { |
| 399 | + if len(metric.GetLabel()) != len(labels) { |
| 400 | + return false |
| 401 | + } |
| 402 | + for _, label := range metric.GetLabel() { |
| 403 | + expectedValue, found := labels[label.GetName()] |
| 404 | + if !found || expectedValue != label.GetValue() { |
| 405 | + return false |
| 406 | + } |
| 407 | + } |
| 408 | + return true |
| 409 | +} |
0 commit comments