|
8 | 8 | "testing" |
9 | 9 | "time" |
10 | 10 |
|
| 11 | + "github.com/prometheus/client_golang/prometheus" |
| 12 | + dto "github.com/prometheus/client_model/go" |
11 | 13 | "github.com/stretchr/testify/require" |
12 | 14 | v1 "k8s.io/api/core/v1" |
13 | 15 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" |
@@ -55,6 +57,23 @@ func TestSolveWithResultReturnsTerminalResultWhenNoTasksToAllocate(t *testing.T) |
55 | 57 | require.False(t, result.EnteredSearch()) |
56 | 58 | } |
57 | 59 |
|
| 60 | +func TestSolveWithResultRecordsNoSearchMetricAsNotAttempted(t *testing.T) { |
| 61 | + labels := map[string]string{ |
| 62 | + "action": "reclaim", |
| 63 | + "result": string(SearchResultNotAttempted), |
| 64 | + "reduced_budget": "false", |
| 65 | + } |
| 66 | + before := scenarioSearchCounterValue(t, "scenario_search_jobs_total", labels) |
| 67 | + solver := NewJobsSolver(nil, nil, nil, framework.Reclaim, nil) |
| 68 | + pendingJob := podgroup_info.NewPodGroupInfo("pending-job") |
| 69 | + |
| 70 | + _, _, _, result := solver.SolveWithResult(&framework.Session{}, pendingJob) |
| 71 | + |
| 72 | + require.Equal(t, SearchResultGeneratorsExhausted, result.Reason()) |
| 73 | + require.False(t, result.EnteredSearch()) |
| 74 | + require.Equal(t, before+1, scenarioSearchCounterValue(t, "scenario_search_jobs_total", labels)) |
| 75 | +} |
| 76 | + |
58 | 77 | func TestSolveWithResultReturnsNoGeneratorWhenGeneratorFuncIsNil(t *testing.T) { |
59 | 78 | ssn, pendingJob := newJobSolverResultTestSession(t, 1) |
60 | 79 | solver := NewJobsSolver(nil, nil, nil, framework.Reclaim, nil) |
@@ -119,6 +138,66 @@ func TestSolveWithResultUsesMinJobBudgetAfterActionBudgetExpired(t *testing.T) { |
119 | 138 | require.False(t, result.EnteredSearch()) |
120 | 139 | } |
121 | 140 |
|
| 141 | +func TestSolveWithResultRecordsGeneratorExhaustedMetricAfterGeneratorAttempt(t *testing.T) { |
| 142 | + labels := map[string]string{ |
| 143 | + "action": "reclaim", |
| 144 | + "result": string(SearchResultGeneratorsExhausted), |
| 145 | + "reduced_budget": "false", |
| 146 | + } |
| 147 | + before := scenarioSearchCounterValue(t, "scenario_search_jobs_total", labels) |
| 148 | + ssn, pendingJob := newJobSolverResultTestSession(t, 1) |
| 149 | + ssn.AddScenarioGenerator("empty", portfolioTestFactory(&portfolioTestGenerator{name: "empty"}), framework.Reclaim) |
| 150 | + solver := NewJobsSolver( |
| 151 | + nil, |
| 152 | + nil, |
| 153 | + func() *utils.JobsOrderByQueues { |
| 154 | + return utils.GetVictimsQueue(ssn, nil) |
| 155 | + }, |
| 156 | + framework.Reclaim, |
| 157 | + nil, |
| 158 | + ) |
| 159 | + |
| 160 | + _, _, _, result := solver.SolveWithResult(ssn, pendingJob) |
| 161 | + |
| 162 | + require.Equal(t, SearchResultGeneratorsExhausted, result.Reason()) |
| 163 | + require.False(t, result.EnteredSearch()) |
| 164 | + require.Equal(t, before+1, scenarioSearchCounterValue(t, "scenario_search_jobs_total", labels)) |
| 165 | +} |
| 166 | + |
| 167 | +func TestSolveWithResultRecordsUnsolvedScenarioDurationAfterSimulation(t *testing.T) { |
| 168 | + generatorName := "test-unsolved-duration" |
| 169 | + labels := map[string]string{ |
| 170 | + "action": "reclaim", |
| 171 | + "generator": generatorName, |
| 172 | + "result": scenarioSearchResultUnsolved, |
| 173 | + } |
| 174 | + before := scenarioSearchHistogramCount(t, "scenario_search_duration_seconds", labels) |
| 175 | + ssn, pendingJob := newJobSolverResultTestSession(t, 1) |
| 176 | + ssn.ClusterInfo.Nodes = map[string]*node_info.NodeInfo{"node-1": {}} |
| 177 | + scenarioToSolve := scenario.NewByNodeScenario( |
| 178 | + ssn, pendingJob, |
| 179 | + podgroup_info.GetTasksToAllocate(pendingJob, ssn.SubGroupOrderFn, ssn.TaskOrderFn, false), |
| 180 | + nil, nil, |
| 181 | + ) |
| 182 | + ssn.AddScenarioGenerator(generatorName, portfolioTestFactory(&portfolioTestGenerator{ |
| 183 | + name: generatorName, |
| 184 | + scenarios: []api.ScenarioInfo{scenarioToSolve}, |
| 185 | + }), framework.Reclaim) |
| 186 | + solver := NewJobsSolver( |
| 187 | + nil, |
| 188 | + nil, |
| 189 | + func() *utils.JobsOrderByQueues { |
| 190 | + return utils.GetVictimsQueue(ssn, nil) |
| 191 | + }, |
| 192 | + framework.Reclaim, |
| 193 | + nil, |
| 194 | + ) |
| 195 | + |
| 196 | + solver.SolveWithResult(ssn, pendingJob) |
| 197 | + |
| 198 | + require.Equal(t, before+1, scenarioSearchHistogramCount(t, "scenario_search_duration_seconds", labels)) |
| 199 | +} |
| 200 | + |
122 | 201 | func TestSolveWithResultRunsCompletePartialSearchForOneGeneratorBeforeNext(t *testing.T) { |
123 | 202 | ssn := newGeneratorTestSession(t, map[string]int{ |
124 | 203 | "node-1": 1, |
@@ -280,3 +359,54 @@ func newJobSolverResultTestSession(t *testing.T, tasksCount int) (*framework.Ses |
280 | 359 | }, |
281 | 360 | }, pendingJob |
282 | 361 | } |
| 362 | + |
| 363 | +func scenarioSearchCounterValue(t *testing.T, metricName string, labels map[string]string) float64 { |
| 364 | + t.Helper() |
| 365 | + |
| 366 | + metric := scenarioSearchMetric(t, metricName, labels) |
| 367 | + if metric == nil || metric.GetCounter() == nil { |
| 368 | + return 0 |
| 369 | + } |
| 370 | + return metric.GetCounter().GetValue() |
| 371 | +} |
| 372 | + |
| 373 | +func scenarioSearchHistogramCount(t *testing.T, metricName string, labels map[string]string) uint64 { |
| 374 | + t.Helper() |
| 375 | + |
| 376 | + metric := scenarioSearchMetric(t, metricName, labels) |
| 377 | + if metric == nil || metric.GetHistogram() == nil { |
| 378 | + return 0 |
| 379 | + } |
| 380 | + return metric.GetHistogram().GetSampleCount() |
| 381 | +} |
| 382 | + |
| 383 | +func scenarioSearchMetric(t *testing.T, metricName string, labels map[string]string) *dto.Metric { |
| 384 | + t.Helper() |
| 385 | + |
| 386 | + families, err := prometheus.DefaultGatherer.Gather() |
| 387 | + require.NoError(t, err) |
| 388 | + for _, family := range families { |
| 389 | + if family.GetName() != metricName { |
| 390 | + continue |
| 391 | + } |
| 392 | + for _, metric := range family.GetMetric() { |
| 393 | + if scenarioSearchMetricHasLabels(metric, labels) { |
| 394 | + return metric |
| 395 | + } |
| 396 | + } |
| 397 | + } |
| 398 | + return nil |
| 399 | +} |
| 400 | + |
| 401 | +func scenarioSearchMetricHasLabels(metric *dto.Metric, labels map[string]string) bool { |
| 402 | + if len(metric.GetLabel()) != len(labels) { |
| 403 | + return false |
| 404 | + } |
| 405 | + for _, label := range metric.GetLabel() { |
| 406 | + expectedValue, found := labels[label.GetName()] |
| 407 | + if !found || expectedValue != label.GetValue() { |
| 408 | + return false |
| 409 | + } |
| 410 | + } |
| 411 | + return true |
| 412 | +} |
0 commit comments