SCRUM-4817 put Model entities into ES #1311
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,74 @@ | ||
| package org.alliancegenome.indexer.indexers; | ||
|
|
||
| import lombok.extern.slf4j.Slf4j; | ||
| import org.alliancegenome.curation_api.model.document.es.AffectedGenomicModelDocument; | ||
| import org.alliancegenome.curation_api.response.SearchResponse; | ||
| import org.alliancegenome.es.util.ProcessDisplayHelper; | ||
| import org.alliancegenome.indexer.config.IndexerConfig; | ||
| import org.alliancegenome.indexer.indexers.curation.service.ModelService; | ||
| import org.apache.commons.collections4.CollectionUtils; | ||
|
|
||
| import java.util.ArrayList; | ||
| import java.util.List; | ||
| import java.util.Set; | ||
| import java.util.concurrent.LinkedBlockingDeque; | ||
|
|
||
| @Slf4j | ||
| public class AffectedGenomicModelIndexer extends Indexer { | ||
|
|
||
| private final ModelService service = new ModelService(); | ||
|
|
||
| public AffectedGenomicModelIndexer(IndexerConfig config) { | ||
| super(config); | ||
| } | ||
|
|
||
| private ProcessDisplayHelper display = new ProcessDisplayHelper(2000); | ||
| private int totalRecords; | ||
|
|
||
| @Override | ||
| public void index() { | ||
| try { | ||
| Set<String> allModelIds = service.getAllModelIds(); | ||
| LinkedBlockingDeque<String> queue = new LinkedBlockingDeque<>(allModelIds); | ||
| display.startProcess("Pulling Affective Genomic Model Documents", queue.size()); | ||
| totalRecords = allModelIds.size(); | ||
| initiateThreading(queue); | ||
| } catch (Exception e) { | ||
| log.error("Error while indexing...", e); | ||
| System.exit(-1); | ||
| } | ||
| } | ||
|
|
||
| protected void startSingleThread(LinkedBlockingDeque<String> queue) { | ||
| List<AffectedGenomicModelDocument> list = new ArrayList<>(); | ||
| while (true) { | ||
| try { | ||
| if (list.size() >= indexerConfig.getBufferSize()) { | ||
| indexDocuments(list); | ||
| list.clear(); | ||
| } | ||
| if (queue.isEmpty()) { | ||
| if (list.size() > 0) { | ||
| indexDocuments(list); | ||
| list.clear(); | ||
| } | ||
| return; | ||
| } | ||
|
|
||
| String key = queue.takeFirst(); | ||
| SearchResponse<AffectedGenomicModelDocument> response = service.getModelDocument(key); | ||
| if (CollectionUtils.isEmpty(response.getResults())) { | ||
| log.debug("No model found for " + key); | ||
| continue; | ||
| } | ||
| list.addAll(response.getResults()); | ||
| display.progressProcess((long) (response.getResults().size())); | ||
| } catch (Exception e) { | ||
| log.error("Error while indexing...", e); | ||
| System.exit(-1); | ||
| return; | ||
| } | ||
| } | ||
| } | ||
|
|
||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,23 @@ | ||
| package org.alliancegenome.indexer.indexers.curation.interfaces; | ||
|
|
||
| import com.fasterxml.jackson.annotation.JsonView; | ||
| import jakarta.ws.rs.*; | ||
| import jakarta.ws.rs.core.MediaType; | ||
| import org.alliancegenome.curation_api.model.document.es.AffectedGenomicModelDocument; | ||
| import org.alliancegenome.curation_api.response.SearchResponse; | ||
| import org.alliancegenome.curation_api.view.View; | ||
|
|
||
| import java.util.HashMap; | ||
|
|
||
| @Path("/model") | ||
| @Produces(MediaType.APPLICATION_JSON) | ||
| @Consumes(MediaType.APPLICATION_JSON) | ||
| public interface GeneModelInterface { | ||
| @POST | ||
| @Path("/gene-documents") | ||
| @JsonView({View.ForPublic.class}) | ||
| SearchResponse<AffectedGenomicModelDocument> findForPublic( | ||
| @DefaultValue("0") @QueryParam("page") Integer page, | ||
| @DefaultValue("10") @QueryParam("limit") Integer limit, | ||
| HashMap<String, Object> params); | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,33 @@ | ||
| package org.alliancegenome.indexer.indexers.curation.service; | ||
|
|
||
| import com.fasterxml.jackson.databind.ObjectMapper; | ||
| import org.alliancegenome.core.config.ConfigHelper; | ||
| import org.alliancegenome.curation_api.model.document.es.AffectedGenomicModelDocument; | ||
| import org.alliancegenome.curation_api.response.SearchResponse; | ||
| import org.alliancegenome.indexer.RestConfig; | ||
| import org.alliancegenome.indexer.indexers.curation.interfaces.GeneModelInterface; | ||
| import si.mazi.rescu.RestProxyFactory; | ||
|
|
||
| import java.util.HashMap; | ||
| import java.util.Set; | ||
|
|
||
| public class ModelService extends BaseService { | ||
|
|
||
| protected ObjectMapper mapper = RestConfig.config.getJacksonObjectMapperFactory().createObjectMapper(); | ||
|
|
||
| private final GeneModelInterface modelApi = RestProxyFactory.createProxy(GeneModelInterface.class, ConfigHelper.getCurationApiUrl(), RestConfig.config); | ||
|
|
||
| public Set<String> getAllGeneIds() { | ||
| return getAllNeoAlleleIDs(); | ||
| } | ||
|
|
||
| public Set<String> getAllModelIds() { | ||
| return getAllNeoModelIDs(); | ||
| } | ||
|
|
||
| public SearchResponse<AffectedGenomicModelDocument> getModelDocument(String modelId) { | ||
| HashMap<String, Object> params = new HashMap<>(); | ||
| params.put("primaryExternalId", modelId); | ||
| return modelApi.findForPublic(0, 1, params); | ||
|
There was a problem hiding this comment. Should we we pulling these in batches from curation. As one at time I would think is going to be slow. |
||
| } | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -95,7 +95,7 @@ public void progressProcess(String data, long amount) { | |
|
|
||
| double percent = 0; | ||
| if (totalSize > 0) { | ||
| percent = ((double) sizeCounter.get() / (double) totalSize); | ||
|
There was a problem hiding this comment. 🚫 [checkstyle] <com.puppycrawl.tools.checkstyle.checks.coding.UnnecessaryParenthesesCheck> reported by reviewdog 🐶 |
||
| } | ||
| long processedAmount = sizeCounter.get() - lastSizeCounter; | ||
| StringBuffer sb = new StringBuffer(this.message); | ||
|
|
||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This needs to be removed and we use the one in curation.