Merge pull request #2624 from opencb/TASK-7645

TASK-7645 - Integrated Clinical Preprocessing, Variant Assessment, and Reporting

Author: pfurio

.gitignore

@@ -20,6 +20,8 @@ make_env
 *.so.*
 *.out
 *.log
+*.error
+*.errors
 *.zip
 # Maven
 target/

opencga-analysis/pom.xml

@@ -287,6 +287,21 @@
                 </exclusion>
             </exclusions>
         </dependency>
+        <dependency>
+            <groupId>de.flapdoodle.embed</groupId>
+            <artifactId>de.flapdoodle.embed.mongo</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>de.flapdoodle.embed</groupId>
+            <artifactId>de.flapdoodle.embed.process</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>de.flapdoodle.reverse</groupId>
+            <artifactId>de.flapdoodle.reverse</artifactId>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>org.yaml</groupId>
             <artifactId>snakeyaml</artifactId>

opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/ClinicalInterpretationManager.java

@@ -314,7 +314,7 @@ public OpenCGAResult<ClinicalVariant> get(Query query, QueryOptions queryOptions
                                 // Only it's updated the following fields
                                 // Important to note that the results include the "new" clinical evidences
                                 clinicalVariant.setComments(primaryFinding.getComments())
-                                        .setFilters(primaryFinding.getFilters())
+                                        .setFilter(primaryFinding.getFilter())
                                         .setDiscussion(primaryFinding.getDiscussion())
                                         .setStatus(primaryFinding.getStatus())
                                         .setAttributes(primaryFinding.getAttributes());

opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/InterpretationAnalysis.java

@@ -20,6 +20,7 @@
 import org.apache.commons.collections4.MapUtils;
 import org.opencb.biodata.models.clinical.ClinicalAnalyst;
 import org.opencb.biodata.models.clinical.interpretation.ClinicalVariant;
+import org.opencb.biodata.models.clinical.interpretation.ClinicalVariantFilter;
 import org.opencb.biodata.models.clinical.interpretation.InterpretationMethod;
 import org.opencb.biodata.models.clinical.interpretation.Software;
 import org.opencb.commons.datastore.core.ObjectMap;
@@ -134,10 +135,10 @@ protected void saveInterpretation(String name, String description, ParamUtils.Sa
                 + SECONDARY_FINDINGS_FILENAME));
 
         for (ClinicalVariant primaryFinding : primaryFindings) {
-            primaryFinding.setFilters(query);
+            primaryFinding.setFilter(new ClinicalVariantFilter(query, GitRepositoryState.getInstance().getBuildVersion(), ""));
         }
         for (ClinicalVariant secondaryFinding : secondaryFindings) {
-            secondaryFinding.setFilters(query);
+            secondaryFinding.setFilter(new ClinicalVariantFilter(query, GitRepositoryState.getInstance().getBuildVersion(), ""));
         }
 
         org.opencb.biodata.models.clinical.interpretation.Interpretation interpretation = new Interpretation()

opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserClinicalVariantCreator.java

@@ -5,10 +5,7 @@
 import org.opencb.biodata.models.clinical.ClinicalAcmg;
 import org.opencb.biodata.models.clinical.ClinicalDiscussion;
 import org.opencb.biodata.models.clinical.ClinicalProperty;
-import org.opencb.biodata.models.clinical.interpretation.ClinicalVariant;
-import org.opencb.biodata.models.clinical.interpretation.ClinicalVariantEvidence;
-import org.opencb.biodata.models.clinical.interpretation.GenomicFeature;
-import org.opencb.biodata.models.clinical.interpretation.VariantClassification;
+import org.opencb.biodata.models.clinical.interpretation.*;
 import org.opencb.biodata.models.variant.Variant;
 import org.opencb.biodata.models.variant.avro.ConsequenceType;
 import org.opencb.biodata.models.variant.avro.GeneCancerAssociation;
@@ -36,8 +33,10 @@ public ClinicalVariant create(Variant variant) {
             return null;
         }
 
-        return new ClinicalVariant(variant.getImpl(), new ArrayList<>(), new ArrayList<>(), new HashMap<>(), new ClinicalDiscussion(),
-                null, ClinicalVariant.Status.NOT_REVIEWED, new ArrayList<>(), new HashMap<>());
+        return new ClinicalVariant(variant.getImpl(), new ArrayList<>(), new ArrayList<>(),
+                new ClinicalVariantFilter(new HashMap<>(), "", ""), new ArrayList<>(), "", new ArrayList<>(), new ClinicalDiscussion(),
+                new ClinicalVariantConfidence(), new ArrayList<>(), ClinicalVariant.Status.NOT_REVIEWED, new ArrayList<>(),
+                new ArrayList<>(), new HashMap<>());
     }
 
     public void addClinicalVariantEvidences(ClinicalVariant clinicalVariant, List<ExomiserTranscriptAnnotation> exomiserTranscripts,

opencga-analysis/src/main/java/org/opencb/opencga/analysis/tools/ToolFactory.java

@@ -41,6 +41,7 @@
 public class ToolFactory {
     private static final Logger logger = LoggerFactory.getLogger(ToolFactory.class);
     private static Map<String, Class<? extends OpenCgaTool>> toolsCache;
+    private static Set<String> toolsCachePackages;
     private static Map<String, Set<Class<? extends OpenCgaTool>>> duplicatedTools;
     private static List<Class<? extends OpenCgaTool>> toolsList;
 
@@ -60,7 +61,7 @@ private static void loadTools(Analysis analysisConf) {
     }
 
     private static synchronized Map<String, Class<? extends OpenCgaTool>> loadTools(List<String> packages) {
-        if (toolsCache == null) {
+        if (isCacheOutdated(packages)) {
             Reflections reflections = new Reflections(new ConfigurationBuilder()
                     .setScanners(
                             new SubTypesScanner(),
@@ -104,9 +105,31 @@ private static synchronized Map<String, Class<? extends OpenCgaTool>> loadTools(
             ToolFactory.toolsList = Collections.unmodifiableList(toolsList);
             ToolFactory.duplicatedTools = Collections.unmodifiableMap(duplicatedTools);
             ToolFactory.toolsCache = cache;
+            // And add packages to the cache
+            if (ToolFactory.toolsCachePackages == null) {
+                ToolFactory.toolsCachePackages = new HashSet<>();
+            }
+            ToolFactory.toolsCachePackages.addAll(packages);
         }
         return toolsCache;
     }
+
+    private static boolean isCacheOutdated(List<String> packages) {
+        if (toolsCache == null || toolsCachePackages == null) {
+            // Cache is empty
+            return true;
+        }
+
+        Set<String> packageSet = new HashSet<>(packages);
+        if (!toolsCachePackages.containsAll(packageSet)) {
+            // There is at least one package missing in the cache
+            return true;
+        }
+
+        // Cache is up to date
+        return false;
+    }
+
     static Collection<URL> getUrlsFromPackages(List<String> packages) {
         Collection<URL> urls = new LinkedList<>();
         for (String pack :packages){

opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantCatalogQueryUtils.java

@@ -199,6 +199,7 @@ public static SegregationMode parse(String name) {
 
     private final StudyFilterValidator studyFilterValidator;
     private final FileFilterValidator fileFilterValidator;
+    private final FileDataValidator fileDataValidator;
     private final SampleFilterValidator sampleFilterValidator;
     private final CohortFilterValidator cohortFilterValidator;
     //    public static final QueryParam SAMPLE_FILTER_GENOTYPE = QueryParam.create("sampleFilterGenotype", "", QueryParam.Type.TEXT_ARRAY);
@@ -208,6 +209,7 @@ public VariantCatalogQueryUtils(CatalogManager catalogManager) {
         super(catalogManager);
         studyFilterValidator = new StudyFilterValidator();
         fileFilterValidator = new FileFilterValidator();
+        fileDataValidator = new FileDataValidator();
         sampleFilterValidator = new SampleFilterValidator();
         cohortFilterValidator = new CohortFilterValidator();
     }
@@ -279,7 +281,7 @@ public Query parseQuery(Query query, QueryOptions queryOptions, CellBaseUtils ce
         sampleFilterValidator.processFilter(query, VariantQueryParam.GENOTYPE, release, token, defaultStudyStr);
         fileFilterValidator.processFilter(query, VariantQueryParam.FILE, release, token, defaultStudyStr);
         fileFilterValidator.processFilter(query, VariantQueryParam.INCLUDE_FILE, release, token, defaultStudyStr);
-        fileFilterValidator.processFilter(query, VariantQueryParam.FILE_DATA, release, token, defaultStudyStr);
+        fileDataValidator.processFilter(query, VariantQueryParam.FILE_DATA, release, token, defaultStudyStr);
         cohortFilterValidator.processFilter(query, VariantQueryParam.COHORT, release, token, defaultStudyStr);
         cohortFilterValidator.processFilter(query, VariantQueryParam.STATS_ALT, release, token, defaultStudyStr);
         cohortFilterValidator.processFilter(query, VariantQueryParam.STATS_REF, release, token, defaultStudyStr);
@@ -950,7 +952,7 @@ private String toIndividualId(String study, String individuaOrSample, String tok
     }
 
     private Region processSampleFilter(Query query, String defaultStudyStr, CellBaseUtils cellBaseUtils,
-                                     ParsedVariantQuery.VariantQueryXref xrefs, String token) throws CatalogException {
+                                       ParsedVariantQuery.VariantQueryXref xrefs, String token) throws CatalogException {
         Region segregationChromosome = null;
         String sampleFilterValue = query.getString(SAMPLE.key());
         if (sampleFilterValue.contains(IS)) {
@@ -1392,10 +1394,18 @@ public static String toStorageFileName(File file) {
     }
 
     public static String toStorageFilePath(File file) {
-        return file.getUri().getPath();
+        switch (file.getType()) {
+            case FILE:
+                return file.getUri().getPath();
+            case VIRTUAL:
+                return file.getPath();
+            case DIRECTORY:
+            default:
+                throw new IllegalArgumentException("Unexpected file type: " + file.getType());
+        }
     }
 
-    public abstract class FilterValidator {
+    public abstract static class FilterValidator {
         protected final QueryOptions RELEASE_OPTIONS = new QueryOptions(INCLUDE, Arrays.asList(
                 FileDBAdaptor.QueryParams.ID.key(),
                 FileDBAdaptor.QueryParams.NAME.key(),
@@ -1557,6 +1567,35 @@ protected List<String> validate(String defaultStudyStr, List<String> values, Int
 
     }
 
+    public class FileDataValidator extends FileFilterValidator {
+        @Override
+        protected void processFilter(Query query, VariantQueryParam param, Integer release, String sessionId, String defaultStudy)
+                throws CatalogException {
+            if (param != VariantQueryParam.FILE_DATA) {
+                throw new IllegalStateException("Unexpected param '" + param + "' in FileDataValidator");
+            }
+            if (VariantQueryUtils.isValidParam(query, param)) {
+                String value = query.getString(FILE_DATA.key());
+                if (!value.contains(IS)) {
+                    // File data does not contain key-value pairs.
+                    // Validation only applies to keys (ie file identifiers)
+                    return;
+                }
+                ParsedQuery<KeyValues<String, KeyOpValue<String, String>>> fileData = parseFileData(query);
+
+                List<String> valuesToValidate = fileData.mapValues(KeyValues::getKey);
+                List<String> validatedValues = validate(defaultStudy, valuesToValidate, release, param, sessionId);
+
+                for (int i = 0; i < fileData.getValues().size(); i++) {
+                    fileData.getValues().get(i).setKey(validatedValues.get(i));
+                }
+
+                String newValue = fileData.toQuery();
+                query.put(param.key(), newValue);
+            }
+        }
+    }
+
     public class SampleFilterValidator extends FilterValidator {
 
         @Override

opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/metadata/CatalogStorageMetadataSynchronizer.java

@@ -37,6 +37,7 @@
 import org.opencb.opencga.catalog.utils.ParamUtils;
 import org.opencb.opencga.core.api.ParamConstants;
 import org.opencb.opencga.core.common.BatchUtils;
+import org.opencb.opencga.core.common.UriUtils;
 import org.opencb.opencga.core.config.storage.CellBaseConfiguration;
 import org.opencb.opencga.core.config.storage.SampleIndexConfiguration;
 import org.opencb.opencga.core.models.cohort.Cohort;
@@ -735,8 +736,8 @@ protected boolean synchronizeFiles(StudyMetadata study, List<File> files, String
             for (Integer virtualFile : virtualFiles) {
                 File file = catalogManager.getFileManager()
                         .get(study.getName(), filePathMap.get(virtualFile), INDEXED_FILES_QUERY_OPTIONS, token).first();
-                boolean annotationIndexReady = annotationReadyFilesFromStorage.contains(file.getUri());
-                boolean secondaryIndexReady = secondaryIndexReadyFilesFromStorage.contains(file.getUri());
+                boolean annotationIndexReady = annotationReadyFilesFromStorage.contains(getFileUri(file));
+                boolean secondaryIndexReady = secondaryIndexReadyFilesFromStorage.contains(getFileUri(file));
                 if (synchronizeIndexedFile(study, file, fileSamplesMap, annotationIndexReady, secondaryIndexReady, token, true)) {
                     modified = true;
                 }
@@ -760,16 +761,16 @@ protected boolean synchronizeFiles(StudyMetadata study, List<File> files, String
                             .iterator(study.getName(), query, INDEXED_FILES_QUERY_OPTIONS, token)) {
                         while (iterator.hasNext()) {
                             File file = iterator.next();
-                            boolean annotationIndexReady = annotationReadyFilesFromStorage.contains(file.getUri());
-                            boolean secondaryIndexReady = secondaryIndexReadyFilesFromStorage.contains(file.getUri());
+                            boolean annotationIndexReady = annotationReadyFilesFromStorage.contains(getFileUri(file));
+                            boolean secondaryIndexReady = secondaryIndexReadyFilesFromStorage.contains(getFileUri(file));
                             if (synchronizeIndexedFile(study, file, fileSamplesMap, annotationIndexReady, secondaryIndexReady, token, true)) {
                                 modifiedFiles++;
                                 modified = true;
                             }
 
                             // Remove processed file from list of uris.
                             // Avoid double processing in case of exception
-                            batch.remove(file.getUri().toString());
+                            batch.remove(getFileUri(file).toString());
                             numFiles++;
                             processedFilesInBatch++;
                             progressLogger.increment(1, modifiedFiles + " updated files");
@@ -818,7 +819,7 @@ protected boolean synchronizeFiles(StudyMetadata study, List<File> files, String
                 .iterator(study.getName(), indexedFilesQuery, INDEXED_FILES_QUERY_OPTIONS, token)) {
             while (iterator.hasNext()) {
                 File file = iterator.next();
-                Integer fileId = fileURIMap.get(file.getUri());
+                Integer fileId = fileURIMap.get(getFileUri(file));
                 if (fileId == null || !indexedFilesFromStorage.contains(fileId)) {
                     // Check for annotation index and secondary annotation index
                     boolean annotationIndexReady;
@@ -827,8 +828,8 @@ protected boolean synchronizeFiles(StudyMetadata study, List<File> files, String
                         annotationIndexReady = false;
                         secondaryIndexReady = false;
                     } else {
-                        annotationIndexReady = annotationReadyFilesFromStorage.contains(file.getUri());
-                        secondaryIndexReady = secondaryIndexReadyFilesFromStorage.contains(file.getUri());
+                        annotationIndexReady = annotationReadyFilesFromStorage.contains(getFileUri(file));
+                        secondaryIndexReady = secondaryIndexReadyFilesFromStorage.contains(getFileUri(file));
                     }
                     synchronizeIndexedFile(study, file, fileSamplesMap, annotationIndexReady, secondaryIndexReady, token, false);
                     modified = true;
@@ -854,7 +855,7 @@ protected boolean synchronizeFiles(StudyMetadata study, List<File> files, String
                 .iterator(study.getName(), runningIndexFilesQuery, INDEXED_FILES_QUERY_OPTIONS, token)) {
             while (iterator.hasNext()) {
                 File file = iterator.next();
-                Integer fileId = fileURIMap.get(file.getUri());
+                Integer fileId = fileURIMap.get(getFileUri(file));
                 FileMetadata fileMetadata;
                 if (fileId == null) {
                     fileMetadata = null;
@@ -893,10 +894,10 @@ protected boolean synchronizeFiles(StudyMetadata study, List<File> files, String
                         modified = true;
                     } else {
                         // Running job. Might be transforming, or have just started. Do not modify the status!
-                        loadingFilesRegardingCatalog.add(file.getUri());
+                        loadingFilesRegardingCatalog.add(getFileUri(file));
                     }
                 } else {
-                    loadingFilesRegardingCatalog.add(file.getUri());
+                    loadingFilesRegardingCatalog.add(getFileUri(file));
                 }
             }
         }
@@ -941,6 +942,14 @@ protected boolean synchronizeFiles(StudyMetadata study, List<File> files, String
         return modified;
     }
 
+    private static URI getFileUri(File file) {
+        if (file.getType() == File.Type.VIRTUAL) {
+            return UriUtils.toUri(file.getPath());
+        } else {
+            return file.getUri();
+        }
+    }
+
     private boolean synchronizeIndexedFile(StudyMetadata study, File file, Map<URI, Set<String>> fileSamplesMap,
                                            boolean annotationIndexReady, boolean secondaryIndexReady, String token, boolean mainIndexReady)
             throws CatalogException {
@@ -996,7 +1005,7 @@ private boolean synchronizeIndexedFile(StudyMetadata study, File file, Map<URI,
             modified = true;
         }
 
-        Set<String> storageSamples = fileSamplesMap.get(file.getUri());
+        Set<String> storageSamples = fileSamplesMap.get(getFileUri(file));
         Set<String> catalogSamples = new HashSet<>(file.getSampleIds());
         if (storageSamples == null) {
             storageSamples = new HashSet<>();