[JN-1664] option to exclude in progress surveys (#1587)

Co-authored-by: Devon <[email protected]>

Author: connorlbark

core/src/main/java/bio/terra/pearl/core/dao/survey/SurveyResponseDao.java

@@ -1,7 +1,6 @@
 package bio.terra.pearl.core.dao.survey;
 
 import bio.terra.pearl.core.dao.BaseMutableJdbiDao;
-import bio.terra.pearl.core.model.file.ParticipantFile;
 import bio.terra.pearl.core.model.file.ScannedParticipantFileDto;
 import bio.terra.pearl.core.model.survey.Answer;
 import bio.terra.pearl.core.model.survey.SurveyResponse;
@@ -34,7 +33,7 @@ public List<SurveyResponse> findByEnrolleeId(UUID enrolleeId) {
     }
 
     /** excludes responses that are associated with removed tasks */
-    public Map<UUID, List<SurveyResponse>> findByEnrolleeIdsNotRemoved(List<UUID> enrolleeIds) {
+    public Map<UUID, List<SurveyResponse>> findByEnrolleeIdsNotRemoved(List<UUID> enrolleeIds, boolean onlyComplete) {
         if (enrolleeIds.isEmpty()) {
             return Collections.emptyMap();
         }
@@ -45,8 +44,9 @@ public Map<UUID, List<SurveyResponse>> findByEnrolleeIdsNotRemoved(List<UUID> en
                                 select sr.* from %s sr
                                 left join participant_task task on task.survey_response_id = sr.id
                                 where sr.enrollee_id in (<enrolleeIds>)
+                                %s
                                 and task.status is distinct from 'REMOVED'
-                                """.formatted(tableName))
+                                """.formatted(tableName, onlyComplete ? "and task.status = 'COMPLETE' and sr.complete = true" : ""))
                         .bindList("enrolleeIds", enrolleeIds)
                         .mapTo(clazz)
                         .stream().collect(Collectors.groupingBy(SurveyResponse::getEnrolleeId, Collectors.toList()))

core/src/main/java/bio/terra/pearl/core/model/export/ExportOptions.java

@@ -3,7 +3,10 @@
 import bio.terra.pearl.core.model.BaseEntity;
 import bio.terra.pearl.core.service.export.ExportFileFormat;
 import com.fasterxml.jackson.annotation.JsonIgnore;
-import lombok.*;
+import lombok.Builder;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
 import lombok.experimental.SuperBuilder;
 
 import java.time.ZoneId;
@@ -20,6 +23,8 @@ public class ExportOptions extends BaseEntity {
     private boolean stableIdsForOptions = false;
     @Builder.Default
     private boolean onlyIncludeMostRecent = true;
+    @Builder.Default
+    private boolean onlyIncludeCompleted = false;
     private String filterString;
     @Builder.Default
     private ExportFileFormat fileFormat = ExportFileFormat.TSV;

core/src/main/java/bio/terra/pearl/core/service/export/EnrolleeExportService.java

@@ -7,7 +7,6 @@
 import bio.terra.pearl.core.model.participant.*;
 import bio.terra.pearl.core.model.search.EnrolleeSearchExpressionResult;
 import bio.terra.pearl.core.model.study.Study;
-import bio.terra.pearl.core.model.study.StudyEnvironment;
 import bio.terra.pearl.core.model.study.StudyEnvironmentConfig;
 import bio.terra.pearl.core.model.survey.*;
 import bio.terra.pearl.core.model.workflow.ParticipantTask;
@@ -92,7 +91,7 @@ public EnrolleeExportService(ProfileService profileService,
     /**
      * exports the specified number of enrollees from the given environment
      * The enrollees will be returned most-recently-created first
-     * */
+     */
     public void export(ExportOptionsWithExpression exportOptions, UUID studyEnvironmentId, OutputStream os) {
 
         List<EnrolleeExportData> enrolleeExportData = loadEnrolleeExportData(studyEnvironmentId, exportOptions);
@@ -217,7 +216,7 @@ public List<EnrolleeExportData> loadEnrolleeExportData(UUID studyEnvironmentId,
         Map<UUID, List<SurveyResponseWithTaskDto>> surveyResponses =
                 attachTasksToSurveyResponses(
                         tasks,
-                        surveyResponseService.findByEnrolleeIdsNotRemoved(enrolleeIds));
+                        surveyResponseService.findByEnrolleeIdsNotRemoved(enrolleeIds, exportOptions.isOnlyIncludeCompleted()));
         Map<UUID, List<KitRequestDto>> kitRequests = kitRequestService.findByEnrollees(enrollees);
 
         return enrollees.stream()

core/src/main/java/bio/terra/pearl/core/service/survey/SurveyResponseService.java

@@ -64,8 +64,8 @@ public List<SurveyResponse> findByEnrolleeId(UUID enrolleeId) {
         return dao.findByEnrolleeId(enrolleeId);
     }
 
-    public Map<UUID, List<SurveyResponse>> findByEnrolleeIdsNotRemoved(List<UUID> enrolleeIds) {
-        return dao.findByEnrolleeIdsNotRemoved(enrolleeIds);
+    public Map<UUID, List<SurveyResponse>> findByEnrolleeIdsNotRemoved(List<UUID> enrolleeIds, boolean onlyComplete) {
+        return dao.findByEnrolleeIdsNotRemoved(enrolleeIds, onlyComplete);
     }
 
     public Optional<SurveyResponse> findOneWithAnswers(UUID responseId) {

core/src/main/resources/db/changelog/changesets/2025_04_11_export_option_only_complete.yaml

@@ -0,0 +1,12 @@
+databaseChangeLog:
+  - changeSet:
+      id: "export_options_only_complete"
+      author: connorlbark
+      changes:
+        - addColumn:
+            tableName: export_options
+            columns:
+              - column:
+                  name: only_include_completed
+                  type: boolean
+                  defaultValueBoolean: false

core/src/main/resources/db/changelog/db.changelog-master.yaml

@@ -401,7 +401,9 @@ databaseChangeLog:
   - include:
       file: changesets/2025_02_14_trigger_admin_email_filter.yaml
       relativeToChangelogFile: true
-
+  - include:
+      file: changesets/2025_04_11_export_option_only_complete.yaml
+      relativeToChangelogFile: true
 
 # README: it is a best practice to put each DDL statement in its own change set. DDL statements
 # are atomic. When they are grouped in a changeset and one fails the changeset cannot be

core/src/test/java/bio/terra/pearl/core/service/export/EnrolleeExportServiceTests.java

@@ -17,8 +17,12 @@
 import bio.terra.pearl.core.model.study.StudyEnvironment;
 import bio.terra.pearl.core.model.study.StudyEnvironmentConfig;
 import bio.terra.pearl.core.model.survey.Survey;
+import bio.terra.pearl.core.model.survey.SurveyResponse;
 import bio.terra.pearl.core.model.survey.SurveyResponseWithTaskDto;
 import bio.terra.pearl.core.model.survey.SurveyType;
+import bio.terra.pearl.core.model.workflow.ParticipantTask;
+import bio.terra.pearl.core.model.workflow.TaskStatus;
+import bio.terra.pearl.core.model.workflow.TaskType;
 import bio.terra.pearl.core.service.export.formatters.ExportFormatUtils;
 import bio.terra.pearl.core.service.export.formatters.item.AnswerItemFormatter;
 import bio.terra.pearl.core.service.export.formatters.item.ItemFormatter;
@@ -745,4 +749,97 @@ public void testExportSubheadersOption(TestInfo testInfo) throws Exception {
         assertThat(export, containsString(",enrollee.createdAt"));
         assertThat(export, containsString(",Created At"));
     }
+
+    @Test
+    @Transactional
+    public void testOnlyCompleteSurveyResponses(TestInfo testInfo) throws Exception {
+
+        ParticipantTask.ParticipantTaskBuilder taskBuilder = ParticipantTask.builder()
+                .status(TaskStatus.NEW)
+                .taskType(TaskType.SURVEY)
+                .targetName("test")
+                .taskOrder(1);
+
+        String testName = getTestName(testInfo);
+        StudyEnvironmentBundle studyEnvBundle = studyEnvironmentFactory.buildBundle(testName, EnvironmentName.sandbox);
+        StudyEnvironment studyEnv = studyEnvBundle.getStudyEnv();
+        PortalEnvironment portalEnv = studyEnvBundle.getPortalEnv();
+        Survey survey = surveyService.create(
+                surveyFactory
+                        .builderWithDependencies(getTestName(testInfo))
+                        .content(SOCIAL_HEALTH_EXCERPT)
+                        .name("Survey Test")
+                        .stableId("examplesurvey")
+                        .surveyType(SurveyType.RESEARCH)
+                        .version(1)
+                        .autoAssign(false)
+                        .build());
+
+        surveyFactory.attachToEnv(survey, studyEnv.getId(), true);
+
+        EnrolleeBundle enrollee1Bundle = enrolleeFactory.buildWithPortalUser(testName, portalEnv, studyEnv, new Profile());
+        Enrollee enrollee1 = enrollee1Bundle.enrollee();
+
+        SurveyResponse e1r1 = surveyResponseFactory.buildWithAnswers(
+                enrollee1,
+                survey,
+                Map.of(
+                        "hd_hd_socialHealth_neighborhoodIsWalkable", "agree"
+                ),
+                true
+        );
+
+        SurveyResponse e1r2 = surveyResponseFactory.buildWithAnswers(
+                enrollee1,
+                survey,
+                Map.of(
+                        "hd_hd_socialHealth_neighborhoodIsWalkable", "disagree"
+                )
+        );
+
+        participantTaskFactory.buildPersisted(enrollee1Bundle,
+                taskBuilder
+                        .surveyResponseId(e1r1.getId())
+                        .status(TaskStatus.COMPLETE));
+        participantTaskFactory.buildPersisted(enrollee1Bundle,
+                taskBuilder
+                        .surveyResponseId(e1r2.getId())
+                        .status(TaskStatus.IN_PROGRESS));
+
+
+        Enrollee enrollee2 = enrolleeFactory.buildPersisted(testName, studyEnv, new Profile());
+
+        SurveyResponse e2r1 = surveyResponseFactory.buildWithAnswers(
+                enrollee2,
+                survey,
+                Map.of(
+                        "hd_hd_socialHealth_neighborhoodIsWalkable", "agree"
+                )
+        );
+
+        participantTaskFactory.buildPersisted(enrollee1Bundle,
+                taskBuilder
+                        .surveyResponseId(e2r1.getId())
+                        .status(TaskStatus.IN_PROGRESS));
+
+
+        ExportOptionsWithExpression exportOptions = new ExportOptionsWithExpression();
+
+        exportOptions.setOnlyIncludeCompleted(true);
+
+        List<EnrolleeExportData> exportData = enrolleeExportService.loadEnrolleeExportData(studyEnv.getId(), exportOptions);
+        List<ModuleFormatter> moduleFormatters = enrolleeExportService.generateModuleInfos(new ExportOptions(), studyEnv.getId(), exportData);
+        List<Map<String, String>> exportMaps = enrolleeExportService.generateExportMaps(exportData, moduleFormatters);
+
+
+        assertThat(exportMaps, hasSize(2));
+
+        Map<String, String> enrollee1Map = exportMaps.stream().filter(map -> map.get("enrollee.shortcode").equals(enrollee1.getShortcode())).findFirst().get();
+        Map<String, String> enrollee2Map = exportMaps.stream().filter(map -> map.get("enrollee.shortcode").equals(enrollee2.getShortcode())).findFirst().get();
+
+        assertThat(enrollee1Map.get("examplesurvey.hd_hd_socialHealth_neighborhoodIsWalkable"), equalTo("Agree"));
+        assertThat(enrollee1Map.containsKey("examplesurvey.hd_hd_socialHealth_neighborhoodIsWalkable[1]"), equalTo(false));
+
+        assertThat(enrollee2Map.containsKey("examplesurvey.hd_hd_socialHealth_neighborhoodIsWalkable"), equalTo(false));
+    }
 }

core/src/test/java/bio/terra/pearl/core/service/workflow/ParticipantTaskServiceTests.java

@@ -11,6 +11,8 @@
 import bio.terra.pearl.core.model.EnvironmentName;
 import bio.terra.pearl.core.model.audit.ResponsibleEntity;
 import bio.terra.pearl.core.model.workflow.ParticipantTask;
+import bio.terra.pearl.core.model.workflow.TaskStatus;
+import bio.terra.pearl.core.model.workflow.TaskType;
 import bio.terra.pearl.core.service.survey.SurveyResponseService;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.TestInfo;

core/src/testFixtures/java/bio/terra/pearl/core/factory/survey/SurveyResponseFactory.java

@@ -4,8 +4,10 @@
 import bio.terra.pearl.core.factory.participant.EnrolleeFactory;
 import bio.terra.pearl.core.model.audit.ResponsibleEntity;
 import bio.terra.pearl.core.model.participant.Enrollee;
-import bio.terra.pearl.core.model.portal.Portal;
-import bio.terra.pearl.core.model.survey.*;
+import bio.terra.pearl.core.model.survey.Answer;
+import bio.terra.pearl.core.model.survey.AnswerType;
+import bio.terra.pearl.core.model.survey.Survey;
+import bio.terra.pearl.core.model.survey.SurveyResponse;
 import bio.terra.pearl.core.model.workflow.HubResponse;
 import bio.terra.pearl.core.model.workflow.ParticipantTask;
 import bio.terra.pearl.core.service.survey.SurveyResponseService;
@@ -15,7 +17,6 @@
 
 import java.util.List;
 import java.util.Map;
-import java.util.UUID;
 
 @Component
 public class SurveyResponseFactory {
@@ -47,12 +48,21 @@ public SurveyResponse buildPersisted(String testName) {
         return surveyResponseService.create(builderWithDependencies(testName).build());
     }
 
-    /** to create a response with an objectValued answer, use a JsonNode value in the answerMap */
+    /**
+     * to create a response with an objectValued answer, use a JsonNode value in the answerMap
+     */
     public SurveyResponse buildWithAnswers(Enrollee enrollee, Survey survey, Map<String, Object> answerMap) {
+        return buildWithAnswers(enrollee, survey, answerMap, false);
+    }
+
+
+    /** to create a response with an objectValued answer, use a JsonNode value in the answerMap */
+    public SurveyResponse buildWithAnswers(Enrollee enrollee, Survey survey, Map<String, Object> answerMap, boolean complete) {
         SurveyResponse response = surveyResponseService.create(SurveyResponse.builder()
                 .enrolleeId(enrollee.getId())
                 .creatingParticipantUserId(enrollee.getParticipantUserId())
                 .surveyId(survey.getId())
+                .complete(complete)
                 .build());
         List<Answer> answers = answerFactory.createFromMap(answerMap, enrollee, survey, response);
         response.setAnswers(answers);

populate/src/main/resources/seed/portals/demo/studies/heartdemo/enrollees/jsalk.json

@@ -53,7 +53,7 @@
         {"questionStableId": "neighborhoodMatrix", "objectValue": "{\"affordable\":5,\"friendlyPeople\":4,\"betterThanOthers\":5,\"easyTravelTo\":3}\n", "viewedLanguage": "en"}
       ],
       "currentPageNo": 1,
-      "complete": true,
+      "complete": false,
       "submittedHoursAgo": 246
     },
     {

ui-admin/src/api/api.tsx

@@ -289,6 +289,7 @@ export type ExportOptions = {
   splitOptionsIntoColumns?: boolean,
   stableIdsForOptions?: boolean,
   onlyIncludeMostRecent?: boolean,
+  onlyIncludeCompleted?: boolean,
   includeSubHeaders?: boolean,
   excludeModules?: string[],
   filterString?: string,