Skip to content

Commit 1bb9433

Browse files
m1chum1chu
committed
Fix: rethrow exceptions in task handlers to enable db-scheduler retry.
1 parent e74be9a commit 1bb9433

5 files changed

Lines changed: 47 additions & 9 deletions

File tree

‎backend/src/main/kotlin/pl/allegro/tech/github/botorchestrator/infra/task/ReactToCheckRunProps.kt‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
package pl.allegro.tech.github.botorchestrator.infra.task
2+
3+
import org.springframework.boot.context.properties.ConfigurationProperties
4+
import java.time.Duration as JavaDuration
5+
6+
@ConfigurationProperties(prefix = "app.tasks.handle-check-run")
7+
data class ReactToCheckRunProps(
8+
val sleepDuration: JavaDuration,
9+
val noSlotsSleepDuration: JavaDuration
10+
)

‎backend/src/main/kotlin/pl/allegro/tech/github/botorchestrator/infra/task/ReactToPullRequestChangeProps.kt‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,5 +5,6 @@ import java.time.Duration as JavaDuration
55

66
@ConfigurationProperties(prefix = "app.tasks.handle-pull-request-change")
77
data class ReactToPullRequestChangeProps(
8-
val sleepDuration: JavaDuration
8+
val sleepDuration: JavaDuration,
9+
val noSlotsSleepDuration: JavaDuration
910
)

‎backend/src/main/kotlin/pl/allegro/tech/github/botorchestrator/infra/task/TasksConfig.kt‎

Lines changed: 27 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@ import com.fasterxml.jackson.databind.ObjectMapper
44
import com.github.kagkarlsson.scheduler.boot.config.DbSchedulerCustomizer
55
import com.github.kagkarlsson.scheduler.serializer.JacksonSerializer
66
import com.github.kagkarlsson.scheduler.serializer.Serializer
7-
import com.github.kagkarlsson.scheduler.task.FailureHandler.OnFailureRetryLater
87
import com.github.kagkarlsson.scheduler.task.helper.OneTimeTask
98
import com.github.kagkarlsson.scheduler.task.helper.Tasks
109
import io.github.oshai.kotlinlogging.KotlinLogging
@@ -14,13 +13,15 @@ import org.springframework.context.annotation.Configuration
1413
import org.springframework.jdbc.core.JdbcTemplate
1514
import pl.allegro.tech.github.botorchestrator.application.CheckRunEventHandler
1615
import pl.allegro.tech.github.botorchestrator.application.PullRequestEventHandler
16+
import kotlin.jvm.optionals.getOrNull
1717
import pl.allegro.tech.github.botorchestrator.domain.NoSlotsAvailable
1818
import pl.allegro.tech.github.botorchestrator.domain.workflows.WorkflowDefinitionsReader
1919
import java.util.Optional
2020

2121
@Configuration
2222
class TasksConfig(
23-
private val props: ReactToPullRequestChangeProps,
23+
private val pullRequestProps: ReactToPullRequestChangeProps,
24+
private val checkRunProps: ReactToCheckRunProps,
2425
private val eventHandler: PullRequestEventHandler,
2526
private val checkRunEventHandler: CheckRunEventHandler,
2627
private val workflowDefinitionsReader: WorkflowDefinitionsReader
@@ -39,7 +40,14 @@ class TasksConfig(
3940
fun reactToPullRequestsChanges(): OneTimeTask<ReactToPullRequestChangeTaskData> {
4041
return Tasks
4142
.oneTime(HANDLE_PULL_REQUEST)
42-
.onFailure(OnFailureRetryLater(props.sleepDuration))
43+
.onFailure { executionComplete, executionOperations ->
44+
val delay = if (executionComplete.cause.getOrNull() is NoSlotsAvailable) {
45+
pullRequestProps.noSlotsSleepDuration
46+
} else {
47+
pullRequestProps.sleepDuration
48+
}
49+
executionOperations.reschedule(executionComplete, executionComplete.timeDone.plus(delay))
50+
}
4351
.onDeadExecutionRevive()
4452
.execute { inst, _ ->
4553
try {
@@ -55,9 +63,11 @@ class TasksConfig(
5563
val workflowConfiguration = workflowDefinitionsReader.getDefinitionFor(inst.data.workflowId)
5664
eventHandler.handle(workflowConfiguration, inst.data.event)
5765
} catch (e: NoSlotsAvailable) {
58-
logger.error(e) { "No available slots for ${inst.taskName}/${inst.id} for PR ${inst.data.event.data.pullRequest.htmlUrl}" }
66+
logger.warn(e) { "No available slots for ${inst.taskName}/${inst.id} for PR ${inst.data.event.data.pullRequest.htmlUrl}, will retry in ${pullRequestProps.noSlotsSleepDuration}" }
67+
throw e
5968
} catch (e: Exception) {
60-
logger.error(e) { "Failed handling ${inst.id} for PR ${inst.data.event.data.pullRequest.htmlUrl}" }
69+
logger.error(e) { "Failed handling ${inst.id} for PR ${inst.data.event.data.pullRequest.htmlUrl}, will retry in ${pullRequestProps.sleepDuration}" }
70+
throw e
6171
} finally {
6272
MDC.remove("pullRequest")
6373
MDC.remove("pullRequestNumber")
@@ -74,7 +84,14 @@ class TasksConfig(
7484
fun reactToCheckRunEvents(): OneTimeTask<ReactToCheckRunTaskData> {
7585
return Tasks
7686
.oneTime(HANDLE_CHECK_RUN)
77-
.onFailure(OnFailureRetryLater(props.sleepDuration))
87+
.onFailure { executionComplete, executionOperations ->
88+
val delay = if (executionComplete.cause.getOrNull() is NoSlotsAvailable) {
89+
checkRunProps.noSlotsSleepDuration
90+
} else {
91+
checkRunProps.sleepDuration
92+
}
93+
executionOperations.reschedule(executionComplete, executionComplete.timeDone.plus(delay))
94+
}
7895
.onDeadExecutionRevive()
7996
.execute { inst, _ ->
8097
try {
@@ -88,9 +105,11 @@ class TasksConfig(
88105
val workflowConfiguration = workflowDefinitionsReader.getDefinitionFor(inst.data.workflowId)
89106
checkRunEventHandler.handle(workflowConfiguration, inst.data.event, inst.data.pullRequestNumber)
90107
} catch (e: NoSlotsAvailable) {
91-
logger.error(e) { "No available slots for ${inst.taskName}/${inst.id} for check_run ${inst.data.event.data.checkRun.id}" }
108+
logger.warn(e) { "No available slots for ${inst.taskName}/${inst.id} for check_run ${inst.data.event.data.checkRun.id}, will retry in ${checkRunProps.noSlotsSleepDuration}" }
109+
throw e
92110
} catch (e: Exception) {
93-
logger.error(e) { "Failed handling ${inst.id} for check_run ${inst.data.event.data.checkRun.id}" }
111+
logger.error(e) { "Failed handling ${inst.id} for check_run ${inst.data.event.data.checkRun.id}, will retry in ${checkRunProps.sleepDuration}" }
112+
throw e
94113
} finally {
95114
MDC.remove("repo")
96115
MDC.remove("checkRunId")

‎backend/src/main/resources/application-integration.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,10 @@ app:
22
tasks:
33
handle-pull-request-change:
44
sleep-duration: 100ms
5+
no-slots-sleep-duration: 1s
6+
handle-check-run:
7+
sleep-duration: 100ms
8+
no-slots-sleep-duration: 1s
59
github:
610
auth:
711
installation-id: 123

‎backend/src/main/resources/application.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,10 @@ app:
99
tasks:
1010
handle-pull-request-change:
1111
sleep-duration: 1s
12+
no-slots-sleep-duration: 30s
13+
handle-check-run:
14+
sleep-duration: 1s
15+
no-slots-sleep-duration: 30s
1216
cleanup:
1317
workflow-run-availability:
1418
interval: PT5M

0 commit comments

Comments
 (0)