Remove duplicate position filter (#9)

* Fix issue with multi-allelic snps in reference panel

* Remove duplicate position filter

Author: lukfor

src/main/java/genepi/imputationserver/steps/fastqc/StatisticsTask.java

@@ -348,21 +348,6 @@ private void processLine(MinimalVariantContext snp, List<SitesEntry> refSnps, in
 			return;
 		}
 
-		// count duplicates
-
-		if ((chunk.lastPos == snp.getStart() && chunk.lastPos > 0)) {
-
-			if (insideChunk) {
-				duplicates++;
-				excludedSnpsWriter.write(snp, "Duplicate");
-				filtered++;
-			}
-
-			chunk.lastPos = snp.getStart();
-			return;
-
-		}
-
 		// update last pos only when not filtered
 		if (!snp.isFiltered()) {
 			chunk.lastPos = snp.getStart();

src/test/java/genepi/imputationserver/steps/QualityControlCommandTest.java

@@ -135,8 +135,8 @@ public void testQcStatisticsAllChunksFailed() throws Exception {
 		OutputReader log = new OutputReader(CLOUDGENE_LOG);
 
 		// check statistics
-		assertTrue(log.hasInMemory("Alternative allele frequency > 0.5 sites: 37,503"));
-		assertTrue(log.hasInMemory("Duplicated sites: 618"));
+		assertTrue(log.hasInMemory("Alternative allele frequency > 0.5 sites: 37,610"));
+		assertTrue(log.hasInMemory("Duplicated sites: 0"));
 		assertTrue(log.hasInMemory("36 Chunk(s) excluded"));
 		assertTrue(log.hasInMemory("No chunks passed the QC step"));
 
@@ -183,8 +183,9 @@ public void testQcStatisticsAllChunksPassed() throws Exception {
 		OutputReader log = new OutputReader(CLOUDGENE_LOG);
 
 		// check statistics
-		assertTrue(log.hasInMemory("Excluded sites in total: 3,058"));
-		assertTrue(log.hasInMemory("Remaining sites in total: 117,498"));
+		log.view();
+		assertTrue(log.hasInMemory("Excluded sites in total: 2,744"));
+		assertTrue(log.hasInMemory("Remaining sites in total: 117,785"));
 
 	}
 
@@ -201,7 +202,7 @@ public void testCountSitesForOneChunkedContig() throws Exception {
 		File[] files = new File(TEST_DATA_TMP).listFiles();
 		Arrays.sort(files);
 		// baseline from a earlier job execution
-		int[] array = { 4750, 5174, 5106, 5832, 5318, 4588, 968, 3002, 5781, 5116, 5699, 6334, 3188 };
+		int[] array = { 4763, 5189, 5123, 5845, 5327, 4595, 970, 3007, 5792, 5129, 5711, 6345, 3195 };
 		int pos = 0;
 
 		for (File file : files) {
@@ -426,8 +427,9 @@ public void testQcStatisticsAllowStrandFlips() throws Exception {
 		OutputReader log = new OutputReader(CLOUDGENE_LOG);
 
 		// check statistics
-		assertTrue(log.hasInMemory("Excluded sites in total: 3,058"));
-		assertTrue(log.hasInMemory("Remaining sites in total: 117,498"));
+		log.view();
+		assertTrue(log.hasInMemory("Excluded sites in total: 2,744"));
+		assertTrue(log.hasInMemory("Remaining sites in total: 117,785"));
 
 	}
 
@@ -443,8 +445,8 @@ public void testQcStatisticsDontAllowStrandFlips() throws Exception {
 		OutputReader log = new OutputReader(CLOUDGENE_LOG);
 
 		// check statistics
-		assertTrue(log.hasInMemory("Excluded sites in total: 3,058"));
-		assertTrue(log.hasInMemory("Remaining sites in total: 117,498"));
+		assertTrue(log.hasInMemory("Excluded sites in total: 2,744"));
+		assertTrue(log.hasInMemory("Remaining sites in total: 117,785"));
 		assertTrue(log.hasInMemory(
 				"<b>Error:</b> More than -1 obvious strand flips have been detected. Please check strand. Imputation cannot be started!"));
 
@@ -462,8 +464,8 @@ public void testQcStatisticsDontAllowAlleleSwitches() throws Exception {
 		OutputReader log = new OutputReader(CLOUDGENE_LOG);
 
 		// check statistics
-		assertTrue(log.hasInMemory("Excluded sites in total: 121,176"));
-		assertTrue(log.hasInMemory("Allele switch: 118,209"));
+		assertTrue(log.hasInMemory("Excluded sites in total: 121,140"));
+		assertTrue(log.hasInMemory("Allele switch: 118,443"));
 		assertTrue(log.hasInMemory("No chunks passed the QC step. Imputation cannot be started!"));
 	}