Handle any exception that is not handled by the client application

integration-tests/src/test/java/uk/gov/hmcts/cp/taskmanager/integration/PostgresIntegrationTestBase.java

@@ -13,6 +13,7 @@ public abstract class PostgresIntegrationTestBase {
 
     public static final String ID_KEY = "id";
     public static final String ATTEMPTS_KEY = "no_of_attempts";
+    public static final String ERROR_KEY = "error";
 
     @Container
     static PostgreSQLContainer<?> postgres =

integration-tests/src/test/java/uk/gov/hmcts/cp/taskmanager/integration/TaskInErrorIntegrationTest.java

@@ -0,0 +1,131 @@
+package uk.gov.hmcts.cp.taskmanager.integration;
+
+import static jakarta.json.Json.createObjectBuilder;
+import static java.time.ZonedDateTime.now;
+import static java.util.UUID.randomUUID;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.awaitility.Awaitility.await;
+import static uk.gov.hmcts.cp.taskmanager.domain.ExecutionStatus.INPROGRESS;
+
+import uk.gov.hmcts.cp.taskmanager.domain.ExecutionInfo;
+import uk.gov.hmcts.cp.taskmanager.domain.ExecutionStatus;
+import uk.gov.hmcts.cp.taskmanager.integration.config.IntegrationTestConfiguration;
+import uk.gov.hmcts.cp.taskmanager.integration.service.TaskStatus;
+import uk.gov.hmcts.cp.taskmanager.integration.service.TaskStatusService;
+import uk.gov.hmcts.cp.taskmanager.persistence.entity.Job;
+import uk.gov.hmcts.cp.taskmanager.persistence.repository.JobsRepository;
+import uk.gov.hmcts.cp.taskmanager.persistence.service.JobService;
+import uk.gov.hmcts.cp.taskmanager.service.ExecutionService;
+
+import java.util.List;
+import java.util.Optional;
+import java.util.UUID;
+
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.test.annotation.DirtiesContext;
+
+@IntegrationTestConfiguration
+@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_EACH_TEST_METHOD)
+
+public class TaskInErrorIntegrationTest extends PostgresIntegrationTestBase {
+
+    @Autowired
+    private ExecutionService executionService;
+
+    @Autowired
+    private JobService jobService;
+
+    @Autowired
+    private JobsRepository jobsRepository;
+
+    @Autowired
+    private JdbcTemplate jdbcTemplate;
+
+    @Autowired
+    private TaskStatusService taskStatusService;
+
+    @AfterEach
+    void tearDown() {
+        jdbcTemplate.execute("delete from JOBS");
+        final Integer jobs = jdbcTemplate.queryForObject("select count(*) from JOBS", Integer.class);
+        assertThat(jobs).isEqualTo(0);
+    }
+
+    @Test
+    void testErrorTaskWithNoRetryAttemptsShouldRunOnlyOnce() throws InterruptedException {
+        // Given - Create a job with error task
+        final UUID taskId = randomUUID();
+        ExecutionInfo executionInfo = new ExecutionInfo(
+                createObjectBuilder()
+                        .add("test", "data")
+                        .add(ID_KEY, taskId.toString())
+                        .add(ERROR_KEY, "error")
+                        .build(),
+                "TEST_ERROR_TASK",
+                now().minusSeconds(1),
+                ExecutionStatus.STARTED,
+                false
+        );
+        executionService.executeWith(executionInfo);
+
+        assertTaskExecutedOnlyOnce(taskId, 1);
+        Thread.sleep(4000);
+        assertTaskExecutedOnlyOnce(taskId, 1);
+
+        // When - Wait for first execution
+        await().atMost(java.time.Duration.ofSeconds(5)).untilAsserted(() -> {
+            List<Job> jobs = jobsRepository.findAll();
+            // Job should still exist (not deleted) because it's status is INPROGRESS
+            assertThat(jobs).isNotEmpty();
+
+            Job job = jobs.get(0);
+            // Retry attempts should be set to 0
+            assertThat(job.getRetryAttemptsRemaining()).isEqualTo(0);
+        });
+    }
+
+    @Test
+    void testErrorRetryTaskWithRetryAttemptsShouldRunAllTheRetryAttempts() throws InterruptedException {
+        // Given - Create a job with error retry task
+        final UUID taskId = randomUUID();
+        ExecutionInfo executionInfo = new ExecutionInfo(
+                createObjectBuilder()
+                        .add("test", "data")
+                        .add(ID_KEY, taskId.toString())
+                        .add(ERROR_KEY, "error")
+                        .build(),
+                "TEST_ERROR_RETRY_TASK",
+                now().minusSeconds(1),
+                ExecutionStatus.STARTED,
+                true
+        );
+        executionService.executeWith(executionInfo);
+
+        assertTaskExecutedOnlyOnce(taskId, 3);
+        Thread.sleep(4000);
+        assertTaskExecutedOnlyOnce(taskId, 3);
+
+        // When - Wait for first execution
+        await().atMost(java.time.Duration.ofSeconds(5)).untilAsserted(() -> {
+            List<Job> jobs = jobsRepository.findAll();
+            // Job should still exist (not deleted) because it's status is INPROGRESS
+            assertThat(jobs).isNotEmpty();
+
+            Job job = jobs.get(0);
+            // Retry attempts should be set to 0
+            assertThat(job.getRetryAttemptsRemaining()).isEqualTo(0);
+        });
+    }
+
+    private void assertTaskExecutedOnlyOnce(final UUID taskId, final int retryAttempts) {
+        await().atMost(java.time.Duration.ofSeconds(5)).untilAsserted(() -> {
+            final Optional<TaskStatus> task = taskStatusService.getById(taskId);
+            assertThat(task.isEmpty()).isFalse();
+            assertThat(task.get().getStatus().equals(INPROGRESS.name())).isTrue();
+            assertThat(task.get().getJobData().getInt(ATTEMPTS_KEY)).isEqualTo(retryAttempts);
+        });
+    }
+}

integration-tests/src/test/java/uk/gov/hmcts/cp/taskmanager/integration/tasks/TestErrorRetryTask.java

@@ -0,0 +1,65 @@
+package uk.gov.hmcts.cp.taskmanager.integration.tasks;
+
+import static java.util.UUID.fromString;
+import static java.util.UUID.randomUUID;
+import static uk.gov.hmcts.cp.taskmanager.domain.ExecutionInfo.executionInfo;
+import static uk.gov.hmcts.cp.taskmanager.domain.ExecutionStatus.COMPLETED;
+import static uk.gov.hmcts.cp.taskmanager.domain.ExecutionStatus.INPROGRESS;
+import static uk.gov.hmcts.cp.taskmanager.integration.PostgresIntegrationTestBase.ERROR_KEY;
+import static uk.gov.hmcts.cp.taskmanager.integration.PostgresIntegrationTestBase.ID_KEY;
+
+import uk.gov.hmcts.cp.taskmanager.domain.ExecutionInfo;
+import uk.gov.hmcts.cp.taskmanager.integration.service.TaskStatusService;
+import uk.gov.hmcts.cp.taskmanager.service.task.ExecutableTask;
+import uk.gov.hmcts.cp.taskmanager.service.task.Task;
+
+import java.util.List;
+import java.util.Optional;
+import java.util.UUID;
+
+import jakarta.json.JsonObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+/**
+ * Test task with retry attempts that fails to complete.
+ * Used for integration testing of tasks that throw unexpected exception during task execution.
+ */
+@Task("TEST_ERROR_RETRY_TASK")
+@Component
+public class TestErrorRetryTask implements ExecutableTask {
+
+    @Autowired
+    private TaskStatusService taskStatusService;
+
+    private static final Logger logger = LoggerFactory.getLogger(TestErrorRetryTask.class);
+
+    @Override
+    public ExecutionInfo execute(ExecutionInfo executionInfo) {
+        final JsonObject jobData = executionInfo.getJobData();
+
+        logger.info("TestErrorTask executing for job: {}", jobData);
+
+        if (jobData.containsKey(ERROR_KEY)) {
+            final UUID id = jobData.containsKey(ID_KEY) ? fromString(jobData.getString(ID_KEY)) : randomUUID();
+            taskStatusService.recordRetryAttempt(id, jobData);
+
+            throw new IllegalStateException("Task with retry attempts failed to complete due to unexpected errors!");
+        }
+
+        return executionInfo().from(executionInfo)
+                .withJobData(jobData)
+                .withExecutionStatus(INPROGRESS)
+                .withShouldRetry(true)
+                .build();
+    }
+
+    @Override
+    public Optional<List<Long>> getRetryDurationsInSecs() {
+        // Return 3 retry attempts with delays: 1s, 2s, 3s
+        return Optional.of(List.of(1L, 2L, 3L));
+    }
+}
+

integration-tests/src/test/java/uk/gov/hmcts/cp/taskmanager/integration/tasks/TestErrorTask.java

@@ -0,0 +1,54 @@
+package uk.gov.hmcts.cp.taskmanager.integration.tasks;
+
+import static java.util.UUID.fromString;
+import static java.util.UUID.randomUUID;
+import static uk.gov.hmcts.cp.taskmanager.domain.ExecutionInfo.executionInfo;
+import static uk.gov.hmcts.cp.taskmanager.domain.ExecutionStatus.COMPLETED;
+import static uk.gov.hmcts.cp.taskmanager.integration.PostgresIntegrationTestBase.ERROR_KEY;
+import static uk.gov.hmcts.cp.taskmanager.integration.PostgresIntegrationTestBase.ID_KEY;
+
+import uk.gov.hmcts.cp.taskmanager.domain.ExecutionInfo;
+import uk.gov.hmcts.cp.taskmanager.integration.service.TaskStatusService;
+import uk.gov.hmcts.cp.taskmanager.service.task.ExecutableTask;
+import uk.gov.hmcts.cp.taskmanager.service.task.Task;
+
+import java.util.UUID;
+
+import jakarta.json.JsonObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+/**
+ * Test task that fails to complete immediately.
+ * Used for integration testing of tasks that throw unexpected exception during task execution.
+ */
+@Task("TEST_ERROR_TASK")
+@Component
+public class TestErrorTask implements ExecutableTask {
+
+    @Autowired
+    private TaskStatusService taskStatusService;
+
+    private static final Logger logger = LoggerFactory.getLogger(TestErrorTask.class);
+
+    @Override
+    public ExecutionInfo execute(ExecutionInfo executionInfo) {
+        final JsonObject jobData = executionInfo.getJobData();
+
+        logger.info("TestErrorTask executing for job: {}", jobData);
+
+        if (jobData.containsKey(ERROR_KEY)) {
+            final UUID id = jobData.containsKey(ID_KEY) ? fromString(jobData.getString(ID_KEY)) : randomUUID();
+            taskStatusService.recordRetryAttempt(id, jobData);
+
+            throw new IllegalStateException("Task failed to complete due to unexpected errors!");
+        }
+
+        return executionInfo().from(executionInfo)
+                .withExecutionStatus(COMPLETED)
+                .build();
+    }
+}
+

task-manager-service/src/main/java/uk/gov/hmcts/cp/taskmanager/domain/executor/TaskExecutor.java

@@ -1,5 +1,6 @@
 package uk.gov.hmcts.cp.taskmanager.domain.executor;
 
+import uk.gov.hmcts.cp.taskmanager.domain.ExecutionStatus;
 import uk.gov.hmcts.cp.taskmanager.service.task.TaskRegistry;
 import uk.gov.hmcts.cp.taskmanager.service.task.ExecutableTask;
 import uk.gov.hmcts.cp.taskmanager.domain.ExecutionInfo;
@@ -206,7 +207,22 @@ private boolean isStartTimeOfTask(final ExecutionInfo executionInfo) {
      * @param executionInfo the execution information, must not be null
      */
     private void executeTask(final ExecutableTask task, final ExecutionInfo executionInfo) {
-        final ExecutionInfo executionResponse = task.execute(executionInfo);
+        ExecutionInfo executionResponse;
+        try{
+            executionResponse = task.execute(executionInfo);
+        } catch (Exception e) {
+            logger.error("Error executing the task: {}; error message: {}; setting task executionStatus to INPROGRESS", job.getAssignedTaskName(), e.getMessage());
+            // When a task is created, its executionStatus is set to STARTED.
+            // If an unhandled exception occurs, the task may continue executing in a loop and never transition to a COMPLETED state.
+            // To prevent this behavior, the client application must handle all unexpected errors.
+            // A generic catch-all block should be implemented to capture any unhandled exceptions and update the executionStatus to INPROGRESS,
+            // allowing the task to exit the loop and complete gracefully.
+            executionResponse = ExecutionInfo.executionInfo()
+                                 .from(executionInfo)
+                                 .withExecutionStatus(ExecutionStatus.INPROGRESS)
+                                 .withShouldRetry(nonNull(job.getRetryAttemptsRemaining()) && job.getRetryAttemptsRemaining() > 0)
+                                 .build();
+        }
 
         if (executionResponse.getExecutionStatus().equals(INPROGRESS)) {
             if (canRetry(task, executionResponse)) {
@@ -221,7 +237,7 @@ private void executeTask(final ExecutableTask task, final ExecutionInfo executio
                     //no retry attempts set
                     //when task is chained from another task; ensure task executed at least once
                     //when same task that was executed; ensure task no more executed
-                    retryAttemptsRemaining = !executionResponse.getAssignedTaskName().equals(job.getAssignedTaskName()) && isNull(retryAttemptsRemaining)
+                    retryAttemptsRemaining = !executionResponse.getAssignedTaskName().equals(job.getAssignedTaskName())
                             ? Integer.valueOf(1)
                             : Integer.valueOf(0);
                 }

task-manager-service/src/test/java/uk/gov/hmcts/cp/taskmanager/domain/executor/TaskExecutorTest.java

@@ -259,7 +259,6 @@ void testRunWithException() {
     @Test
     void testRunWithExceptionOnRelease() {
         when(taskRegistry.getTask("TEST_TASK")).thenReturn(Optional.of(executableTask));
-        when(executableTask.execute(any(ExecutionInfo.class))).thenThrow(new RuntimeException("Task execution failed"));
         doThrow(new RuntimeException("Release failed")).when(jobService).releaseJob(any(UUID.class));
 
         // Should not throw exception, should handle gracefully