Merge branch '2.1.3-merge'

mogoodrich · mogoodrich · commit 002150a16c1e · 2019-03-13T15:32:02.000-04:00
diff --git a/scripts/rpm/bahmni-batch b/scripts/rpm/bahmni-batch
@@ -2,6 +2,24 @@
 
 . /etc/bahmni-installer/bahmni.conf
 
+export JOB_NAME=""
+export EXTERNAL_COHORT_TYPES=""
+if [[ -z $1 ]]; then
+    export JOB_NAME="Total Export"
+else
+    INPUT_TYPE="$(cut -d'=' -f1 <<< $1)"
+    if [[ "$INPUT_TYPE" == "belongs_to_external_cohort" ]]; then
+        export EXTERNAL_COHORT_TYPES="$(cut -d'=' -f2 <<< $1)"
+    fi
+    if [[ -z $2 ]]; then
+    export JOB_NAME="External Cohort Export"
+    else
+        INPUT_TYPE="$(cut -d'=' -f1 <<< $2)"
+        if [[ "$INPUT_TYPE" == "job_name" ]]; then
+            export JOB_NAME="$(cut -d'=' -f2 <<< $2)"
+        fi
+    fi
+fi
 echo "Running bahmni-batch..."
 java -DOPENMRS_DB_SERVER=${OPENMRS_DB_SERVER} -DOPENMRS_DB_USER=${OPENMRS_DB_USERNAME} -DOPENMRS_DB_PASSWORD=${OPENMRS_DB_PASSWORD} -jar /opt/bahmni-batch/libs/bahmni-batch-*.jar >> /opt/bahmni-batch/log/bahmni-batch.log
 echo "Done"
diff --git a/src/main/java/org/bahmni/batch/BatchConfiguration.java b/src/main/java/org/bahmni/batch/BatchConfiguration.java
@@ -34,8 +34,6 @@
 @EnableBatchProcessing
 public class BatchConfiguration extends DefaultBatchConfigurer {
 
-	public static final String FULL_DATA_EXPORT_JOB_NAME = "endtbExports";
-
 	@Autowired
 	private JobBuilderFactory jobBuilderFactory;
 
@@ -69,6 +67,9 @@ public class BatchConfiguration extends DefaultBatchConfigurer {
 	@Value("${bahmniConfigFolder}")
 	private Resource bahmniConfigFolder;
 
+	@Value("${jobName}")
+	private String jobName;
+
 	@Autowired
 	private ReportGenerator reportGenerator;
 
@@ -82,7 +83,7 @@ public Job completeDataExport() throws IOException {
 
 		List<BahmniForm> forms = formListProcessor.retrieveAllForms();
 
-		FlowBuilder<FlowJobBuilder> completeDataExport = jobBuilderFactory.get(FULL_DATA_EXPORT_JOB_NAME)
+		FlowBuilder<FlowJobBuilder> completeDataExport = jobBuilderFactory.get(jobName)
 				.incrementer(new RunIdIncrementer()).preventRestart()
 				.listener(listener())
 		                .flow(treatmentRegistrationBaseExportStep.getStep())
diff --git a/src/main/java/org/bahmni/batch/BatchUtils.java b/src/main/java/org/bahmni/batch/BatchUtils.java
@@ -7,6 +7,7 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 
 public class BatchUtils {
@@ -33,4 +34,26 @@ public static List<String> convertConceptNamesToSet(String conceptNames){
 		return conceptNamesSet;
 	}
 
+    private static String constructSqlWithParameter(String sql, String parameter, String value) {
+        return sql.replaceAll(String.format(":%s", parameter), value);
+    }
+
+    public static String constructSqlWithCohortParameters(String sql, List<String> externalCohortTypes){
+		Boolean belongsToExternalCohort = externalCohortTypes.size() >= 1 && !externalCohortTypes.get(0).isEmpty();
+		String cohortTypesAsString = getCohortTypesAsString(externalCohortTypes);
+		sql = constructSqlWithParameter(sql, "belongsToExternalCohort",
+				belongsToExternalCohort.toString());
+		sql = constructSqlWithParameter(sql, "externalCohortTypes", cohortTypesAsString);
+		return sql;
+	}
+
+	private static String getCohortTypesAsString(List<String> cohortTypes) {
+		List<String> cohortTypesWithQuotes = new ArrayList<>();
+		for (String cohortType : cohortTypes) {
+			cohortTypesWithQuotes.add("\"" + cohortType + "\"");
+		}
+		String cohortTypesAsString = Arrays.toString(cohortTypesWithQuotes.toArray());
+		cohortTypesAsString = String.format("%s", cohortTypesAsString.substring(1, cohortTypesAsString.length() - 1));
+		return cohortTypesAsString;
+	}
 }
diff --git a/src/main/java/org/bahmni/batch/JobCompletionNotificationListener.java b/src/main/java/org/bahmni/batch/JobCompletionNotificationListener.java
@@ -34,12 +34,15 @@ public JobCompletionNotificationListener(@Value("${outputFolder}") Resource outp
 	    this.zipFolder = zipFolder;
     }
 
+	@Value("${jobName}")
+	private String jobName;
+
     @Override
     public void beforeJob(JobExecution jobExecution) {
 	    try {
 		    FileUtils.deleteQuietly(outputFolder.getFile());
 		    FileUtils.forceMkdir(outputFolder.getFile());
-		    String zipFileName = BatchConfiguration.FULL_DATA_EXPORT_JOB_NAME + new SimpleDateFormat("yyyyMMddHHmm").format(new Date()) + ".zip";
+		    String zipFileName = jobName + new SimpleDateFormat("yyyyMMddHHmm").format(new Date()) + ".zip";
 		    jobExecution.getExecutionContext().put(OUTPUT_FILE_NAME_CONTEXT_KEY,zipFileName);
 	    }
 	    catch (IOException e) {
diff --git a/src/main/java/org/bahmni/batch/ReportGenerator.java b/src/main/java/org/bahmni/batch/ReportGenerator.java
@@ -10,6 +10,7 @@
 
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
+import java.util.Comparator;
 import java.util.List;
 
 @Component
@@ -24,7 +25,11 @@ public class ReportGenerator {
 	private SimpleDateFormat dateFormat = new SimpleDateFormat("dd-MMM-yyyy HH:mm");
 
 	public String generateReport(){
-		List<JobInstance> jobInstanceList = jobExplorer.findJobInstancesByJobName(BatchConfiguration.FULL_DATA_EXPORT_JOB_NAME,0,20);
+		List<String> jobNames = jobExplorer.getJobNames();
+		List<JobInstance> jobInstanceList = new ArrayList<>();
+		for (String jobName : jobNames) {
+			jobInstanceList.addAll(jobExplorer.findJobInstancesByJobName(jobName, 0, 20));
+		}
 		List<JobResult> results = transformJobExecutionsToReport(getJobExecutionsForInstances(jobInstanceList));
 
 		return evaluator.evaluate("report.ftl",results);
@@ -40,15 +45,25 @@ private List<JobExecution> getJobExecutionsForInstances(List<JobInstance> jobIns
 
 	private List<JobResult> transformJobExecutionsToReport(List<JobExecution> jobExecutions) {
 		List<JobResult> jobResults = new ArrayList<>();
+		sortAllJobExecutions(jobExecutions);
 		for(JobExecution execution: jobExecutions){
 			String zipFileName = execution.getExecutionContext().getString(JobCompletionNotificationListener.OUTPUT_FILE_NAME_CONTEXT_KEY);
 			jobResults.add(new JobResult(dateFormat.format(execution.getCreateTime()),
-					execution.getExitStatus().getExitCode(), zipFileName));
+					execution.getExitStatus().getExitCode(), zipFileName, execution.getJobInstance().getJobName()));
 		}
 
 		return jobResults;
 	}
 
+	private void sortAllJobExecutions(List<JobExecution> jobExecutions) {
+		jobExecutions.sort(new Comparator<JobExecution>() {
+			@Override
+			public int compare(JobExecution o1, JobExecution o2) {
+				return o2.getCreateTime().compareTo(o1.getCreateTime());
+			}
+		});
+	}
+
 	public void setJobExplorer(JobExplorer jobExplorer) {
 		this.jobExplorer = jobExplorer;
 	}
diff --git a/src/main/java/org/bahmni/batch/exports/BaseExportStep.java b/src/main/java/org/bahmni/batch/exports/BaseExportStep.java
@@ -1,6 +1,5 @@
 package org.bahmni.batch.exports;
 
-import org.bahmni.batch.BatchUtils;
 import org.springframework.batch.core.Step;
 import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
 import org.springframework.batch.item.database.JdbcCursorItemReader;
@@ -15,9 +14,14 @@
 import javax.sql.DataSource;
 import java.io.IOException;
 import java.io.Writer;
+import java.util.List;
 
+import static org.bahmni.batch.BatchUtils.constructSqlWithCohortParameters;
+import static org.bahmni.batch.BatchUtils.convertResourceOutputToString;
 
 public class BaseExportStep {
+    private List<String> externalCohortTypes;
+
     private DataSource dataSource;
 
     private StepBuilderFactory stepBuilderFactory;
@@ -32,13 +36,15 @@ public class BaseExportStep {
 
     private String sql;
 
-    public BaseExportStep(StepBuilderFactory stepBuilderFactory, DataSource dataSource, Resource sqlResource, Resource outputFolder, String exportName, String headers) {
+    public BaseExportStep(StepBuilderFactory stepBuilderFactory, DataSource dataSource, Resource sqlResource,
+                          Resource outputFolder, String exportName, String headers, List<String> externalCohortTypes) {
         this.dataSource = dataSource;
         this.stepBuilderFactory = stepBuilderFactory;
         this.sqlResource = sqlResource;
         this.outputFolder = outputFolder;
         this.exportName = exportName;
         this.headers = headers;
+        this.externalCohortTypes = externalCohortTypes;
     }
 
     public Step getStep() {
@@ -80,6 +86,7 @@ public String getHeaders() {
 
     @PostConstruct
     public void postConstruct(){
-        this.sql = BatchUtils.convertResourceOutputToString(sqlResource);
+        String sql = convertResourceOutputToString(sqlResource);
+        this.sql = constructSqlWithCohortParameters(sql, externalCohortTypes);
     }
 }
diff --git a/src/main/java/org/bahmni/batch/exports/MetaDataCodeDictionaryExportStep.java b/src/main/java/org/bahmni/batch/exports/MetaDataCodeDictionaryExportStep.java
@@ -7,6 +7,7 @@
 import org.springframework.stereotype.Component;
 
 import javax.sql.DataSource;
+import java.util.List;
 
 @Component
 public class MetaDataCodeDictionaryExportStep extends BaseExportStep {
@@ -15,8 +16,9 @@ public MetaDataCodeDictionaryExportStep(StepBuilderFactory stepBuilderFactory,
                                             DataSource dataSource,
                                             @Value("classpath:sql/metaDataCodeDictionary.sql") Resource sqlResource,
                                             @Value("${outputFolder}/metaDataCodeDictionary.csv") Resource outputFolder,
-                                            @Value("${metaDataCodeDictionaryHeaders}") String headers) {
-        super(stepBuilderFactory, dataSource, sqlResource, outputFolder, "metaDataCodeDictionary", headers);
+                                            @Value("${metaDataCodeDictionaryHeaders}") String headers,
+                                            @Value("#{'${externalCohortTypes}'.split(',')}") List<String> externalCohortTypes) {
+        super(stepBuilderFactory, dataSource, sqlResource, outputFolder, "metaDataCodeDictionary", headers, externalCohortTypes);
     }
 
 }
diff --git a/src/main/java/org/bahmni/batch/exports/NonTBDrugOrderBaseExportStep.java b/src/main/java/org/bahmni/batch/exports/NonTBDrugOrderBaseExportStep.java
@@ -7,6 +7,7 @@
 import org.springframework.stereotype.Component;
 
 import javax.sql.DataSource;
+import java.util.List;
 
 @Component
 public class NonTBDrugOrderBaseExportStep extends BaseExportStep {
@@ -16,7 +17,8 @@ public NonTBDrugOrderBaseExportStep(StepBuilderFactory stepBuilderFactory,
                                         DataSource dataSource,
                                         @Value("classpath:sql/nonTbDrugOrder.sql") Resource sqlResource,
                                         @Value("${outputFolder}/nonTbDrugOrder.csv") Resource outputFolder,
-                                        @Value("${nonTbDrugOrderHeaders}") String headers ) {
-        super(stepBuilderFactory, dataSource, sqlResource, outputFolder, "nonTbDrugOrder", headers);
+                                        @Value("${nonTbDrugOrderHeaders}") String headers,
+                                        @Value("#{'${externalCohortTypes}'.split(',')}") List<String> externalCohortTypes) {
+        super(stepBuilderFactory, dataSource, sqlResource, outputFolder, "nonTbDrugOrder", headers, externalCohortTypes);
     }
 }
diff --git a/src/main/java/org/bahmni/batch/exports/TBDrugOrderBaseExportStep.java b/src/main/java/org/bahmni/batch/exports/TBDrugOrderBaseExportStep.java
@@ -1,25 +1,23 @@
 package org.bahmni.batch.exports;
 
-
 import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.core.io.Resource;
 import org.springframework.stereotype.Component;
-
 import javax.sql.DataSource;
+import java.util.List;
+
 @Component
 public class TBDrugOrderBaseExportStep extends  BaseExportStep{
 
-
     @Autowired
     public TBDrugOrderBaseExportStep(StepBuilderFactory stepBuilderFactory,
                                      DataSource dataSource,
                                      @Value("classpath:sql/tbDrugOrder.sql") Resource sqlResource,
                                      @Value("${outputFolder}/tbDrugOrder.csv") Resource outputFolder,
-                                     @Value("${tbDrugOrderHeaders}")String headers) {
-        super(stepBuilderFactory, dataSource, sqlResource, outputFolder, "tbDrugOrder", headers);
+                                     @Value("${tbDrugOrderHeaders}")String headers,
+                                     @Value("#{'${externalCohortTypes}'.split(',')}") List<String> externalCohortTypes) {
+        super(stepBuilderFactory, dataSource, sqlResource, outputFolder, "tbDrugOrder", headers, externalCohortTypes);
     }
-
-
 }
diff --git a/src/main/java/org/bahmni/batch/exports/TreatmentRegistrationBaseExportStep.java b/src/main/java/org/bahmni/batch/exports/TreatmentRegistrationBaseExportStep.java
@@ -8,6 +8,7 @@
 import org.springframework.stereotype.Component;
 
 import javax.sql.DataSource;
+import java.util.List;
 
 @Component
 public class TreatmentRegistrationBaseExportStep extends BaseExportStep {
@@ -16,7 +17,8 @@ public class TreatmentRegistrationBaseExportStep extends BaseExportStep {
     public TreatmentRegistrationBaseExportStep(StepBuilderFactory stepBuilderFactory, DataSource dataSource,
                                                @Value("classpath:sql/treatmentRegistration.sql") Resource sqlResource,
                                                @Value("${outputFolder}/treatmentRegistration.csv") Resource outputFolder,
-                                               @Value("${treatmentRegistrationHeaders}")String headers ) {
-        super(stepBuilderFactory, dataSource, sqlResource, outputFolder, "treatmentRegistration", headers);
+                                               @Value("${treatmentRegistrationHeaders}")String headers,
+                                               @Value("#{'${externalCohortTypes}'.split(',')}") List<String> externalCohortTypes) {
+        super(stepBuilderFactory, dataSource, sqlResource, outputFolder, "treatmentRegistration", headers, externalCohortTypes);
     }
 }
diff --git a/src/main/java/org/bahmni/batch/form/ObservationProcessor.java b/src/main/java/org/bahmni/batch/form/ObservationProcessor.java
@@ -42,6 +42,9 @@ public class ObservationProcessor implements ItemProcessor<Map<String,Object>, L
 	@Value("classpath:sql/leafObs.sql")
 	private Resource leafObsSqlResource;
 
+	@Value("#{'${externalCohortTypes}'.split(',')}")
+	private List<String> externalCohortTypes;
+
 	@Autowired
 	private FormFieldTransformer formFieldTransformer;
 
@@ -60,13 +63,15 @@ public List<Obs> process(Map<String,Object> obsRow) throws Exception {
 
 		setObsIdAndParentObsId(obsRows,(Integer)obsRow.get("obs_id"), (Integer)obsRow.get("parent_obs_id"));
 
-		return obsRows;
+		return obsRows.size() > 0 ? obsRows : null;
 	}
 
 	private List<Obs> fetchAllLeafObs(List<Integer> allChildObsGroupIds) {
-		Map<String,List<Integer>> params = new HashMap<>();
+		Map<String, Object> params = new HashMap<>();
 		params.put("childObsIds",allChildObsGroupIds);
 		params.put("leafConceptIds",formFieldTransformer.transformFormToFieldIds(form));
+		params.put("belongsToExternalCohort", externalCohortTypes.size() >= 1 && !externalCohortTypes.get(0).isEmpty());
+		params.put("externalCohortTypes", externalCohortTypes);
 
 		return jdbcTemplate.query(leafObsSql, params, new BeanPropertyRowMapper<Obs>(Obs.class){
 			@Override
@@ -119,6 +124,10 @@ public void setFormFieldTransformer(FormFieldTransformer formFieldTransformer) {
 		this.formFieldTransformer = formFieldTransformer;
 	}
 
+	public void setExternalCohortTypes(List<String> externalCohortTypes) {
+		this.externalCohortTypes = externalCohortTypes;
+	}
+
 	@PostConstruct
 	public void postConstruct(){
 		this.obsDetailSql = BatchUtils.convertResourceOutputToString(obsDetailSqlResource);
diff --git a/src/main/java/org/bahmni/batch/report/JobResult.java b/src/main/java/org/bahmni/batch/report/JobResult.java
@@ -5,11 +5,13 @@ public class JobResult {
 	private String dateOfExecution;
 	private String status;
 	private String zipFileName;
+	private String jobName;
 
-	public JobResult(String dateOfExecution, String status, String zipFileName) {
+	public JobResult(String dateOfExecution, String status, String zipFileName, String jobName) {
 		this.dateOfExecution = dateOfExecution;
 		this.status = status;
 		this.zipFileName = zipFileName;
+		this.jobName = jobName;
 	}
 
 	public String getDateOfExecution() {
@@ -35,4 +37,8 @@ public String getZipFileName() {
 	public void setZipFileName(String zipFileName) {
 		this.zipFileName = zipFileName;
 	}
+
+	public String getJobName() {
+		return jobName;
+	}
 }
diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties
@@ -9,6 +9,8 @@ outputFolder=file:/opt/bahmni-batch/temp
 zipFolder=file:/home/bahmni/endtb_export
 templates=file:src/main/resources/templates
 bahmniConfigFolder=file:/var/www/bahmni_config
+externalCohortTypes=${EXTERNAL_COHORT_TYPES}
+jobName=${JOB_NAME}
 
 treatmentRegistrationHeaders=id_emr,dob,age,sex,tbregtype,regnum,d_reg,reg_facility,status,patient_id,patient_program_id,date_created,date_changed
 tbDrugOrderHeaders=regnum,tbdrug,tbdose,unit,tbroute,tbdrugfreq,d_tbdrugstart,addlinstr,d_tbdrugend,reas_tbd_stop,id_aenum_reas_d_stop_oth,date_created,date_changed
diff --git a/src/main/resources/sql/leafObs.sql b/src/main/resources/sql/leafObs.sql
@@ -8,6 +8,11 @@ select o.concept_id as conceptId,
 from patient_program pp
 join program_attribute_type pg_at on (pg_at.name = 'Registration Number')
 left join  patient_program_attribute ppa  on (pp.patient_program_id = ppa.patient_program_id and pg_at.program_attribute_type_id=ppa.attribute_type_id)
+  left join program_attribute_type pg_at_cohort on (pg_at_cohort.name = 'Belongs to external cohort')
+  left join patient_program_attribute ppa_cohort on (pp.patient_program_id = ppa_cohort.patient_program_id and
+                                                     pg_at_cohort.program_attribute_type_id =
+                                                     ppa_cohort.attribute_type_id)
+  left join concept_view ppa_cv on ppa_cv.concept_id = ppa_cohort.value_reference AND ppa_cv.retired IS FALSE
 join  episode_patient_program epp on (pp.patient_program_id = epp.patient_program_id)
 join episode_encounter ep_en on (epp.episode_id = ep_en.episode_id)
 join   obs o on (ep_en.encounter_id = o.encounter_id)
@@ -17,6 +22,8 @@ left outer join concept_reference_term_map_view cv on (cv.concept_id = codedConc
 left outer join concept_view cvn on (codedConcept.concept_id = cvn.concept_id)
 where
   o.obs_id in (:childObsIds)
-  and obs_con.concept_id  in (:leafConceptIds)
-  and o.voided=0
-  AND pp.voided = 0;
+  and obs_con.concept_id in (:leafConceptIds) AND
+  o.voided = 0
+  AND pp.voided = 0
+  AND ((:belongsToExternalCohort IS TRUE AND ppa_cv.concept_full_name IN (:externalCohortTypes))
+       OR (:belongsToExternalCohort IS FALSE));
diff --git a/src/main/resources/sql/nonTbDrugOrder.sql b/src/main/resources/sql/nonTbDrugOrder.sql
diff --git a/src/main/resources/sql/tbDrugOrder.sql b/src/main/resources/sql/tbDrugOrder.sql
diff --git a/src/main/resources/sql/treatmentRegistration.sql b/src/main/resources/sql/treatmentRegistration.sql
diff --git a/src/main/resources/templates/report.ftl b/src/main/resources/templates/report.ftl
diff --git a/src/test/java/org/bahmni/batch/BatchUtilsTest.java b/src/test/java/org/bahmni/batch/BatchUtilsTest.java
diff --git a/src/test/java/org/bahmni/batch/ReportGeneratorTest.java b/src/test/java/org/bahmni/batch/ReportGeneratorTest.java
diff --git a/src/test/java/org/bahmni/batch/form/ObservationProcessorTest.java b/src/test/java/org/bahmni/batch/form/ObservationProcessorTest.java
