Closes #113. Fix large concept set export

Author: anthonysena

src/main/java/org/ohdsi/webapi/conceptset/ConceptSetExport.java

@@ -23,10 +23,7 @@ public class ConceptSetExport {
 
     @JsonProperty("ConceptSetExpression")
     public ConceptSetExpression csExpression;
-    
-    @JsonProperty("ConceptIds")
-    public Collection<Long> conceptIds;
-    
+        
     @JsonProperty("IdentifierConcepts")
     public Collection<Concept> identifierConcepts;
 

src/main/java/org/ohdsi/webapi/service/CohortDefinitionService.java

@@ -543,20 +543,10 @@ private ArrayList<ConceptSetExport> getConceptSetExports(CohortDefinition def, S
       export.ConceptSetName = cs.name;
       export.csExpression = cs.expression;
 
-      // Resolve the concept set
-      export.conceptIds = vocabService.resolveConceptSetExpression(vocabSource.sourceKey, export.csExpression);
-
-      // Create an array of concept Ids that will be used in the subsequent calls
-      long[] conceptIds = new long[export.conceptIds.size()];
-      Iterator<Long> iter = export.conceptIds.iterator();
-      for (int j = 0; iter.hasNext(); j++) {
-        conceptIds[j] = iter.next();
-      }
-
       // Lookup the identifiers
-      export.identifierConcepts = vocabService.executeIdentifierLookup(vocabSource.sourceKey, conceptIds);
+      export.identifierConcepts = vocabService.executeIncludedConceptLookup(vocabSource.sourceKey, cs.expression);
       // Lookup the mapped items
-      export.mappedConcepts = vocabService.executeMappedLookup(vocabSource.sourceKey, conceptIds);
+      export.mappedConcepts = vocabService.executeMappedLookup(vocabSource.sourceKey, cs.expression);
 
       exports.add(export);
     }

src/main/java/org/ohdsi/webapi/service/ConceptSetService.java

@@ -222,22 +222,10 @@ private ConceptSetExport getConceptSetForExport(int conceptSetId, SourceInfo voc
         // Get the concept set expression
         cs.csExpression = this.getConceptSetExpression(conceptSetId);
 
-        // Resolve the concept set
-        cs.conceptIds = vocabService.resolveConceptSetExpression(vocabSource.sourceKey, cs.csExpression);
-
-        // Create an array of concept Ids that will be used in the subsequent calls
-        long[] conceptIds = new long[cs.conceptIds.size()];
-        Iterator<Long> iter = cs.conceptIds.iterator();
-        for (int j = 0; iter.hasNext(); j++) {
-            conceptIds[j] = iter.next();
-        }
-        //Java 8 this will be more efficent:
-        //long[] conceptIds = cs.conceptIds.stream().mapToLong(i->i).toArray();
-
         // Lookup the identifiers
-        cs.identifierConcepts = vocabService.executeIdentifierLookup(vocabSource.sourceKey, conceptIds);
+        cs.identifierConcepts = vocabService.executeIncludedConceptLookup(vocabSource.sourceKey, cs.csExpression); //vocabService.executeIdentifierLookup(vocabSource.sourceKey, conceptIds);
         // Lookup the mapped items
-        cs.mappedConcepts = vocabService.executeMappedLookup(vocabSource.sourceKey, conceptIds);
+        cs.mappedConcepts = vocabService.executeMappedLookup(vocabSource.sourceKey, cs.csExpression);
 
         return cs;
     }

src/main/java/org/ohdsi/webapi/service/VocabularyService.java

@@ -93,6 +93,24 @@ public Collection<Concept> executeIdentifierLookup(@PathParam("sourceKey") Strin
 
     return getSourceJdbcTemplate(source).query(sql_statement, this.rowMapper);
   }
+  
+  public Collection<Concept> executeIncludedConceptLookup(String sourceKey, ConceptSetExpression conceptSetExpression) {
+    Source source = getSourceRepository().findBySourceKey(sourceKey);
+    String tableQualifier = source.getTableQualifier(SourceDaimon.DaimonType.Vocabulary);
+
+    ConceptSetExpressionQueryBuilder builder = new ConceptSetExpressionQueryBuilder();
+    String query = builder.buildExpressionQuery(conceptSetExpression);
+
+    query = SqlRender.renderSql(query, new String[]{"cdm_database_schema"}, new String[]{tableQualifier});
+    
+    String sql_statement = ResourceHelper.GetResourceAsString("/resources/vocabulary/sql/lookupIdentifiers.sql");
+    sql_statement = SqlRender.renderSql(sql_statement, new String[]{"identifiers", "CDM_schema"}, new String[]{
+      query, tableQualifier});
+    sql_statement = SqlTranslate.translateSql(sql_statement, "sql server", source.getSourceDialect());
+
+    return getSourceJdbcTemplate(source).query(sql_statement, this.rowMapper);
+  }
+  
 
   /**
    * @summary Lookup source codes in the specified vocabulary
@@ -151,6 +169,23 @@ public Collection<Concept> executeMappedLookup(@PathParam("sourceKey") String so
     return getSourceJdbcTemplate(source).query(sql_statement, this.rowMapper);
   }
 
+    public Collection<Concept> executeMappedLookup(String sourceKey, ConceptSetExpression conceptSetExpression) {
+    Source source = getSourceRepository().findBySourceKey(sourceKey);
+    String tableQualifier = source.getTableQualifier(SourceDaimon.DaimonType.Vocabulary);
+
+    ConceptSetExpressionQueryBuilder builder = new ConceptSetExpressionQueryBuilder();
+    String query = builder.buildExpressionQuery(conceptSetExpression);
+
+    query = SqlRender.renderSql(query, new String[]{"cdm_database_schema"}, new String[]{tableQualifier});
+
+    String sql_statement = ResourceHelper.GetResourceAsString("/resources/vocabulary/sql/getMappedSourcecodes.sql");
+    sql_statement = SqlRender.renderSql(sql_statement, new String[]{"identifiers", "CDM_schema"}, new String[]{
+      query, tableQualifier});
+    sql_statement = SqlTranslate.translateSql(sql_statement, "sql server", source.getSourceDialect());
+
+    return getSourceJdbcTemplate(source).query(sql_statement, this.rowMapper);
+  }
+
   @Path("search")
   @POST
   @Produces(MediaType.APPLICATION_JSON)