|
8 | 8 | "testing" |
9 | 9 | "time" |
10 | 10 |
|
| 11 | + "github.com/prometheus/client_golang/prometheus" |
| 12 | + dto "github.com/prometheus/client_model/go" |
11 | 13 | "github.com/stretchr/testify/require" |
12 | 14 | v1 "k8s.io/api/core/v1" |
13 | 15 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" |
@@ -47,6 +49,22 @@ func TestSolveWithResultReturnsTerminalResultWhenNoTasksToAllocate(t *testing.T) |
47 | 49 | require.False(t, result.ReducedBudget()) |
48 | 50 | } |
49 | 51 |
|
| 52 | +func TestSolveWithResultRecordsNoSearchMetricAsNotAttempted(t *testing.T) { |
| 53 | + labels := map[string]string{ |
| 54 | + "action": "reclaim", |
| 55 | + "result": string(SearchResultNotAttempted), |
| 56 | + "reduced_budget": "false", |
| 57 | + } |
| 58 | + before := scenarioSearchCounterValue(t, "scenario_search_jobs_total", labels) |
| 59 | + solver := NewJobsSolver(nil, nil, nil, framework.Reclaim, nil) |
| 60 | + pendingJob := podgroup_info.NewPodGroupInfo("pending-job") |
| 61 | + |
| 62 | + _, _, _, result := solver.SolveWithResult(&framework.Session{}, pendingJob) |
| 63 | + |
| 64 | + require.Equal(t, SearchResultGeneratorsExhausted, result.Reason()) |
| 65 | + require.Equal(t, before+1, scenarioSearchCounterValue(t, "scenario_search_jobs_total", labels)) |
| 66 | +} |
| 67 | + |
50 | 68 | func TestSolveWithResultReturnsNoGeneratorWhenGeneratorFuncIsNil(t *testing.T) { |
51 | 69 | ssn, pendingJob := newJobSolverResultTestSession(t, 1) |
52 | 70 | solver := NewJobsSolver(nil, nil, nil, framework.Reclaim, nil) |
@@ -151,6 +169,65 @@ func TestSolveWithResultReportsDeadlineWhenBudgetExhaustsDuringScenarioSearch(t |
151 | 169 | require.Equal(t, SearchResultDeadlineExhausted, result.Reason()) |
152 | 170 | } |
153 | 171 |
|
| 172 | +func TestSolveWithResultRecordsGeneratorExhaustedMetricAfterGeneratorAttempt(t *testing.T) { |
| 173 | + labels := map[string]string{ |
| 174 | + "action": "reclaim", |
| 175 | + "result": string(SearchResultGeneratorsExhausted), |
| 176 | + "reduced_budget": "false", |
| 177 | + } |
| 178 | + before := scenarioSearchCounterValue(t, "scenario_search_jobs_total", labels) |
| 179 | + ssn, pendingJob := newJobSolverResultTestSession(t, 1) |
| 180 | + ssn.AddScenarioGenerator("empty", portfolioTestFactory(&portfolioTestGenerator{name: "empty"})) |
| 181 | + solver := NewJobsSolver( |
| 182 | + nil, |
| 183 | + nil, |
| 184 | + func() *utils.JobsOrderByQueues { |
| 185 | + return utils.GetVictimsQueue(ssn, nil) |
| 186 | + }, |
| 187 | + framework.Reclaim, |
| 188 | + nil, |
| 189 | + ) |
| 190 | + |
| 191 | + _, _, _, result := solver.SolveWithResult(ssn, pendingJob) |
| 192 | + |
| 193 | + require.Equal(t, SearchResultGeneratorsExhausted, result.Reason()) |
| 194 | + require.Equal(t, before+1, scenarioSearchCounterValue(t, "scenario_search_jobs_total", labels)) |
| 195 | +} |
| 196 | + |
| 197 | +func TestSolveWithResultRecordsUnsolvedScenarioDurationAfterSimulation(t *testing.T) { |
| 198 | + generatorName := "test-unsolved-duration" |
| 199 | + labels := map[string]string{ |
| 200 | + "action": "reclaim", |
| 201 | + "generator": generatorName, |
| 202 | + "result": scenarioSearchResultUnsolved, |
| 203 | + } |
| 204 | + before := scenarioSearchHistogramCount(t, "scenario_search_duration_seconds", labels) |
| 205 | + ssn, pendingJob := newJobSolverResultTestSession(t, 1) |
| 206 | + ssn.ClusterInfo.Nodes = map[string]*node_info.NodeInfo{"node-1": {}} |
| 207 | + scenarioToSolve := scenario.NewByNodeScenario( |
| 208 | + ssn, pendingJob, |
| 209 | + podgroup_info.GetTasksToAllocate(pendingJob, ssn.SubGroupOrderFn, ssn.TaskOrderFn, false), |
| 210 | + nil, nil, |
| 211 | + ) |
| 212 | + ssn.AddScenarioGenerator(generatorName, portfolioTestFactory(&portfolioTestGenerator{ |
| 213 | + name: generatorName, |
| 214 | + scenarios: []api.ScenarioInfo{scenarioToSolve}, |
| 215 | + })) |
| 216 | + solver := NewJobsSolver( |
| 217 | + nil, |
| 218 | + nil, |
| 219 | + func() *utils.JobsOrderByQueues { |
| 220 | + return utils.GetVictimsQueue(ssn, nil) |
| 221 | + }, |
| 222 | + framework.Reclaim, |
| 223 | + nil, |
| 224 | + ) |
| 225 | + |
| 226 | + solver.SolveWithResult(ssn, pendingJob) |
| 227 | + |
| 228 | + require.Equal(t, before+1, scenarioSearchHistogramCount(t, "scenario_search_duration_seconds", labels)) |
| 229 | +} |
| 230 | + |
154 | 231 | func TestSolveWithResultRunsCompletePartialSearchForOneGeneratorBeforeNext(t *testing.T) { |
155 | 232 | ssn := newGeneratorTestSession(t, map[string]int{ |
156 | 233 | "node-1": 1, |
@@ -268,3 +345,54 @@ func newJobSolverResultTestSession(t *testing.T, tasksCount int) (*framework.Ses |
268 | 345 | }, |
269 | 346 | }, pendingJob |
270 | 347 | } |
| 348 | + |
| 349 | +func scenarioSearchCounterValue(t *testing.T, metricName string, labels map[string]string) float64 { |
| 350 | + t.Helper() |
| 351 | + |
| 352 | + metric := scenarioSearchMetric(t, metricName, labels) |
| 353 | + if metric == nil || metric.GetCounter() == nil { |
| 354 | + return 0 |
| 355 | + } |
| 356 | + return metric.GetCounter().GetValue() |
| 357 | +} |
| 358 | + |
| 359 | +func scenarioSearchHistogramCount(t *testing.T, metricName string, labels map[string]string) uint64 { |
| 360 | + t.Helper() |
| 361 | + |
| 362 | + metric := scenarioSearchMetric(t, metricName, labels) |
| 363 | + if metric == nil || metric.GetHistogram() == nil { |
| 364 | + return 0 |
| 365 | + } |
| 366 | + return metric.GetHistogram().GetSampleCount() |
| 367 | +} |
| 368 | + |
| 369 | +func scenarioSearchMetric(t *testing.T, metricName string, labels map[string]string) *dto.Metric { |
| 370 | + t.Helper() |
| 371 | + |
| 372 | + families, err := prometheus.DefaultGatherer.Gather() |
| 373 | + require.NoError(t, err) |
| 374 | + for _, family := range families { |
| 375 | + if family.GetName() != metricName { |
| 376 | + continue |
| 377 | + } |
| 378 | + for _, metric := range family.GetMetric() { |
| 379 | + if scenarioSearchMetricHasLabels(metric, labels) { |
| 380 | + return metric |
| 381 | + } |
| 382 | + } |
| 383 | + } |
| 384 | + return nil |
| 385 | +} |
| 386 | + |
| 387 | +func scenarioSearchMetricHasLabels(metric *dto.Metric, labels map[string]string) bool { |
| 388 | + if len(metric.GetLabel()) != len(labels) { |
| 389 | + return false |
| 390 | + } |
| 391 | + for _, label := range metric.GetLabel() { |
| 392 | + expectedValue, found := labels[label.GetName()] |
| 393 | + if !found || expectedValue != label.GetValue() { |
| 394 | + return false |
| 395 | + } |
| 396 | + } |
| 397 | + return true |
| 398 | +} |
0 commit comments