Skip to content

Commit 6f0f681

Browse files
MalinskiyMalinskiy
committed
fix(core): don't schedule retries according to RetryStrategy if test reached terminal state
1 parent 7a75392 commit 6f0f681

File tree

9 files changed

+149
-20
lines changed

9 files changed

+149
-20
lines changed

‎core/src/main/kotlin/com/malinskiy/marathon/execution/progress/PoolProgressAccumulator.kt

+2-2
Original file line numberDiff line numberDiff line change
@@ -219,7 +219,7 @@ class PoolProgressAccumulator(
219219
transitionTo(TestState.Failed(total = total, running = running - 1, done = done))
220220
}
221221
on<TestEvent.AddRetry> {
222-
transitionTo(TestState.Failed(total = total + 1, running = running, done = done))
222+
transitionTo(TestState.Failed(total = total, running = running, done = done), TestAction.Complete)
223223
}
224224
on<TestEvent.RemoveAttempts> {
225225
transitionTo(TestState.Failed(total = total - it.count, running = running, done = done))
@@ -239,7 +239,7 @@ class PoolProgressAccumulator(
239239
transitionTo(TestState.Passed(total = total, running = running - 1, done = done))
240240
}
241241
on<TestEvent.AddRetry> {
242-
transitionTo(TestState.Passed(total = total + 1, running = running, done = done))
242+
transitionTo(TestState.Passed(total = total, running = running, done = done), TestAction.Complete)
243243
}
244244
on<TestEvent.RemoveAttempts> {
245245
transitionTo(TestState.Passed(total = total - it.count, running = running, done = done))

‎core/src/main/kotlin/com/malinskiy/marathon/execution/queue/QueueActor.kt

+1-2
Original file line numberDiff line numberDiff line change
@@ -186,10 +186,9 @@ class QueueActor(
186186
device: DeviceInfo
187187
) {
188188
logger.debug { "handle failed tests ${device.serialNumber}" }
189-
val retryList = retryStrategy.process(poolId, failed, testShard)
189+
val retryList = retryStrategy.process(poolId, failed, testShard, poolProgressAccumulator)
190190

191191
retryList.forEach {
192-
poolProgressAccumulator.retryTest(it.test)
193192
val testAction = poolProgressAccumulator.testEnded(device, it)
194193
processTestAction(testAction, it)
195194
rerunTest(it.test)

‎core/src/main/kotlin/com/malinskiy/marathon/execution/strategy/RetryStrategy.kt

+7-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,13 @@ package com.malinskiy.marathon.execution.strategy
33
import com.malinskiy.marathon.device.DevicePoolId
44
import com.malinskiy.marathon.execution.TestResult
55
import com.malinskiy.marathon.execution.TestShard
6+
import com.malinskiy.marathon.execution.progress.PoolProgressAccumulator
67

78
interface RetryStrategy {
8-
fun process(devicePoolId: DevicePoolId, tests: Collection<TestResult>, testShard: TestShard): List<TestResult>
9+
fun process(
10+
devicePoolId: DevicePoolId,
11+
tests: Collection<TestResult>,
12+
testShard: TestShard,
13+
poolProgressAccumulator: PoolProgressAccumulator
14+
): List<TestResult>
915
}

‎core/src/main/kotlin/com/malinskiy/marathon/execution/strategy/impl/retry/NoRetryStrategy.kt

+7-1
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,16 @@ package com.malinskiy.marathon.execution.strategy.impl.retry
33
import com.malinskiy.marathon.device.DevicePoolId
44
import com.malinskiy.marathon.execution.TestResult
55
import com.malinskiy.marathon.execution.TestShard
6+
import com.malinskiy.marathon.execution.progress.PoolProgressAccumulator
67
import com.malinskiy.marathon.execution.strategy.RetryStrategy
78

89
class NoRetryStrategy : RetryStrategy {
9-
override fun process(devicePoolId: DevicePoolId, tests: Collection<TestResult>, testShard: TestShard): List<TestResult> {
10+
override fun process(
11+
devicePoolId: DevicePoolId,
12+
tests: Collection<TestResult>,
13+
testShard: TestShard,
14+
poolProgressAccumulator: PoolProgressAccumulator
15+
): List<TestResult> {
1016
return emptyList()
1117
}
1218

‎core/src/main/kotlin/com/malinskiy/marathon/execution/strategy/impl/retry/fixedquota/FixedQuotaRetryStrategy.kt

+14-2
Original file line numberDiff line numberDiff line change
@@ -4,17 +4,29 @@ import com.malinskiy.marathon.config.strategy.RetryStrategyConfiguration
44
import com.malinskiy.marathon.device.DevicePoolId
55
import com.malinskiy.marathon.execution.TestResult
66
import com.malinskiy.marathon.execution.TestShard
7+
import com.malinskiy.marathon.execution.progress.PoolProgressAccumulator
8+
import com.malinskiy.marathon.execution.queue.TestAction
79
import com.malinskiy.marathon.execution.strategy.RetryStrategy
810

911
class FixedQuotaRetryStrategy(private val cnf: RetryStrategyConfiguration.FixedQuotaRetryStrategyConfiguration) : RetryStrategy {
1012
private val retryWatchdog = RetryWatchdog(cnf.totalAllowedRetryQuota, cnf.retryPerTestQuota)
1113
private val poolTestCaseFailureAccumulator = PoolTestFailureAccumulator()
1214

13-
override fun process(devicePoolId: DevicePoolId, tests: Collection<TestResult>, testShard: TestShard): List<TestResult> {
15+
override fun process(
16+
devicePoolId: DevicePoolId,
17+
tests: Collection<TestResult>,
18+
testShard: TestShard,
19+
poolProgressAccumulator: PoolProgressAccumulator
20+
): List<TestResult> {
1421
return tests.filter { testResult ->
1522
poolTestCaseFailureAccumulator.record(devicePoolId, testResult.test)
1623
val flakinessResultCount = testShard.flakyTests.count { it == testResult.test }
17-
retryWatchdog.requestRetry(poolTestCaseFailureAccumulator.getCount(devicePoolId, testResult.test) + flakinessResultCount)
24+
val failuresCount = poolTestCaseFailureAccumulator.getCount(devicePoolId, testResult.test) + flakinessResultCount
25+
if (retryWatchdog.retryPossible(failuresCount) && poolProgressAccumulator.retryTest(testResult.test) != TestAction.Complete) {
26+
retryWatchdog.requestRetry(failuresCount)
27+
} else {
28+
false
29+
}
1830
}
1931
}
2032

‎core/src/main/kotlin/com/malinskiy/marathon/execution/strategy/impl/retry/fixedquota/RetryWatchdog.kt

+6
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,12 @@ internal class RetryWatchdog(
1515
return totalAllowedRetryAvailable && singleTestAllowed
1616
}
1717

18+
fun retryPossible(failuresCount: Int): Boolean {
19+
val totalAllowedRetryAvailable = totalAllowedRetryLeft.get() >= 0
20+
val singleTestAllowed = failuresCount <= maxRetryPerTestQuota
21+
return totalAllowedRetryAvailable && singleTestAllowed
22+
}
23+
1824
private fun totalAllowedRetryAvailable(): Boolean {
1925
return totalAllowedRetryLeft.decrementAndGet() >= 0
2026
}

‎core/src/test/kotlin/com/malinskiy/marathon/execution/progress/PoolProgressAccumulatorTest.kt

+7-7
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import com.malinskiy.marathon.execution.TestShard
1212
import com.malinskiy.marathon.execution.TestStatus
1313
import com.malinskiy.marathon.generateTest
1414
import com.malinskiy.marathon.report.getDevice
15+
import org.amshove.kluent.shouldBe
1516
import org.mockito.kotlin.mock
1617
import org.mockito.kotlin.reset
1718
import org.amshove.kluent.shouldBeEqualTo
@@ -727,17 +728,16 @@ class PoolProgressAccumulatorTest {
727728
* test 2 failed
728729
*/
729730
reporter.testStarted(device, test2)
730-
reporter.testEnded(device, TestResult(test2, device, "2", TestStatus.FAILURE, 1, 2))
731-
reporter.progress().shouldBeEqualTo(2 / 3f)
732-
733731
/**
734732
* adding 4 retries for test2 and then test 2 passes once
735733
*/
736-
reporter.retryTest(test2)
737-
reporter.retryTest(test2)
738-
reporter.retryTest(test2)
739-
reporter.retryTest(test2)
734+
reporter.retryTest(test2) shouldBe null
735+
reporter.retryTest(test2) shouldBe null
736+
reporter.retryTest(test2) shouldBe null
737+
reporter.retryTest(test2) shouldBe null
738+
reporter.testEnded(device, TestResult(test2, device, "2", TestStatus.FAILURE, 1, 2))
740739
reporter.progress().shouldBeEqualTo(2 / 7f)
740+
741741
reporter.testStarted(device, test2)
742742
reporter.testEnded(device, TestResult(test2, device, "2", TestStatus.PASSED, 2, 3))
743743
reporter.progress().shouldBeEqualTo(3 / 7f)

‎core/src/test/kotlin/com/malinskiy/marathon/execution/strategy/impl/retry/NoRetryStrategyTest.kt

+28-1
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,48 @@
11
package com.malinskiy.marathon.execution.strategy.impl.retry
22

3+
import com.malinskiy.marathon.analytics.internal.pub.Track
4+
import com.malinskiy.marathon.config.Configuration
5+
import com.malinskiy.marathon.config.strategy.ExecutionMode
6+
import com.malinskiy.marathon.config.strategy.ExecutionStrategyConfiguration
7+
import com.malinskiy.marathon.config.vendor.VendorConfiguration
38
import com.malinskiy.marathon.device.DevicePoolId
49
import com.malinskiy.marathon.execution.TestShard
10+
import com.malinskiy.marathon.execution.progress.PoolProgressAccumulator
511
import com.malinskiy.marathon.generateTestResults
612
import com.malinskiy.marathon.generateTests
713
import org.amshove.kluent.shouldBeEmpty
814
import org.junit.jupiter.api.Test
15+
import org.mockito.kotlin.mock
16+
import java.io.File
917

1018
class NoRetryStrategyTest {
19+
private val anySuccessConfig = Configuration.Builder(
20+
name = "",
21+
outputDir = File("")
22+
).apply {
23+
vendorConfiguration = VendorConfiguration.StubVendorConfiguration
24+
debug = false
25+
analyticsTracking = false
26+
executionStrategy = ExecutionStrategyConfiguration(ExecutionMode.ANY_SUCCESS, fast = false)
27+
}.build()
28+
29+
private val track = mock<Track>()
30+
1131
@Test
1232
fun `should return empty list`() {
1333
val tests = generateTests(50)
1434
val testResults = generateTestResults(tests)
1535
val strategy = NoRetryStrategy()
1636
val devicePoolId = DevicePoolId("devicePoolId")
1737
val testShard = TestShard(tests)
18-
val result = strategy.process(devicePoolId, testResults, testShard)
38+
val accumulator = PoolProgressAccumulator(
39+
devicePoolId,
40+
TestShard(tests),
41+
anySuccessConfig,
42+
track
43+
)
44+
45+
val result = strategy.process(devicePoolId, testResults, testShard, accumulator)
1946
result.shouldBeEmpty()
2047
}
2148
}

‎core/src/test/kotlin/com/malinskiy/marathon/execution/strategy/impl/retry/fixedquota/FixedQuotaRetryStrategyTest.kt

+77-4
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,51 @@
11
package com.malinskiy.marathon.execution.strategy.impl.retry.fixedquota
22

3+
import com.malinskiy.marathon.analytics.internal.pub.Track
4+
import com.malinskiy.marathon.config.Configuration
5+
import com.malinskiy.marathon.config.strategy.ExecutionMode
6+
import com.malinskiy.marathon.config.strategy.ExecutionStrategyConfiguration
37
import com.malinskiy.marathon.config.strategy.RetryStrategyConfiguration
8+
import com.malinskiy.marathon.config.vendor.VendorConfiguration
49
import com.malinskiy.marathon.device.DevicePoolId
10+
import com.malinskiy.marathon.execution.TestResult
511
import com.malinskiy.marathon.execution.TestShard
12+
import com.malinskiy.marathon.execution.progress.PoolProgressAccumulator
613
import com.malinskiy.marathon.extension.toRetryStrategy
714
import com.malinskiy.marathon.generateTestResults
815
import com.malinskiy.marathon.generateTests
16+
import com.malinskiy.marathon.report.getDevice
917
import org.amshove.kluent.shouldBe
1018
import org.junit.jupiter.api.Test
19+
import org.mockito.kotlin.mock
20+
import java.io.File
1121

1222
class FixedQuotaRetryStrategyTest {
23+
private val anySuccessConfig = Configuration.Builder(
24+
name = "",
25+
outputDir = File("")
26+
).apply {
27+
vendorConfiguration = VendorConfiguration.StubVendorConfiguration
28+
debug = false
29+
analyticsTracking = false
30+
executionStrategy = ExecutionStrategyConfiguration(ExecutionMode.ANY_SUCCESS, fast = false)
31+
}.build()
32+
33+
private val track = mock<Track>()
1334

1435
@Test
1536
fun `total quota tests, total quota is 1`() {
1637
val strategy = RetryStrategyConfiguration.FixedQuotaRetryStrategyConfiguration(totalAllowedRetryQuota = 1).toRetryStrategy()
1738
val poolId = DevicePoolId("DevicePoolId-1")
1839
val tests = generateTests(10)
1940
val testResults = generateTestResults(tests)
20-
strategy.process(poolId, testResults, TestShard(tests)).size shouldBe 1
41+
val accumulator = PoolProgressAccumulator(
42+
poolId,
43+
TestShard(tests),
44+
anySuccessConfig,
45+
track
46+
)
47+
48+
strategy.process(poolId, testResults, TestShard(tests), accumulator).size shouldBe 1
2149
}
2250

2351
@Test
@@ -26,7 +54,13 @@ class FixedQuotaRetryStrategyTest {
2654
val poolId = DevicePoolId("DevicePoolId-1")
2755
val tests = generateTests(10)
2856
val testResults = generateTestResults(tests)
29-
strategy.process(poolId, testResults, TestShard(tests)).size shouldBe 10
57+
val accumulator = PoolProgressAccumulator(
58+
poolId,
59+
TestShard(tests),
60+
anySuccessConfig,
61+
track
62+
)
63+
strategy.process(poolId, testResults, TestShard(tests), accumulator).size shouldBe 10
3064
}
3165

3266
@Test
@@ -35,8 +69,14 @@ class FixedQuotaRetryStrategyTest {
3569
val poolId = DevicePoolId("DevicePoolId-1")
3670
val tests = generateTests(50)
3771
val testResults = generateTestResults(tests)
72+
val accumulator = PoolProgressAccumulator(
73+
poolId,
74+
TestShard(tests),
75+
anySuccessConfig,
76+
track
77+
)
3878

39-
strategy.process(poolId, testResults, TestShard(tests)).size shouldBe 50
79+
strategy.process(poolId, testResults, TestShard(tests), accumulator).size shouldBe 50
4080
}
4181

4282
@Test
@@ -45,11 +85,44 @@ class FixedQuotaRetryStrategyTest {
4585
val poolId = DevicePoolId("DevicePoolId-1")
4686
val tests = generateTests(50)
4787
val testResults = generateTestResults(tests)
88+
val accumulator = PoolProgressAccumulator(
89+
poolId,
90+
TestShard(tests),
91+
anySuccessConfig,
92+
track
93+
)
94+
95+
strategy.process(
96+
poolId,
97+
testResults,
98+
TestShard(tests, flakyTests = tests + tests + tests),
99+
accumulator
100+
).size shouldBe 0
101+
}
102+
103+
@Test
104+
fun `should return 0 tests if test reached terminal state`() {
105+
val strategy = RetryStrategyConfiguration.FixedQuotaRetryStrategyConfiguration().toRetryStrategy()
106+
val poolId = DevicePoolId("DevicePoolId-1")
107+
val tests = generateTests(1)
108+
val testResults = generateTestResults(tests)
109+
val accumulator = PoolProgressAccumulator(
110+
poolId,
111+
TestShard(tests),
112+
anySuccessConfig,
113+
track
114+
)
115+
val deviceInfo = getDevice()
116+
117+
val test = tests.first()
118+
accumulator.testStarted(deviceInfo, test)
119+
accumulator.testEnded(deviceInfo, testResults.first())
48120

49121
strategy.process(
50122
poolId,
51123
testResults,
52-
TestShard(tests, flakyTests = tests + tests + tests)
124+
TestShard(tests),
125+
accumulator
53126
).size shouldBe 0
54127
}
55128
}

0 commit comments

Comments
 (0)