diff --git a/api/pom.xml b/api/pom.xml index 72c4568a38..4d923f433d 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -102,6 +102,11 @@ guava 32.0.0-jre + + co.elastic.clients + elasticsearch-java + 8.9.0 + org.apache.httpcomponents fluent-hc diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/modules/searchEngine/SearchQuery.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/modules/searchEngine/SearchQuery.java index 9ea30e1b97..895be67a41 100644 --- a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/modules/searchEngine/SearchQuery.java +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/modules/searchEngine/SearchQuery.java @@ -161,4 +161,8 @@ public enum Order { * @return defines whether the text of a facet field should be compared case insensitively. */ boolean isFacetTextCompareCaseInsensitive(); + + void setSimpleQuery(boolean value); + + boolean isSimpleQuery(); } diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/search/controller/PagedSearchController.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/search/controller/PagedSearchController.java index 63a652f89f..6ef6172938 100644 --- a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/search/controller/PagedSearchController.java +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/search/controller/PagedSearchController.java @@ -172,7 +172,7 @@ public static ResponseValues process(VitroRequest vreq, Map int hitsPerPage = getHitsPerPage(vreq); int documentsToReturn = hitsPerPage; if (!wasHtmlRequested) { - documentsToReturn = getDocumentsNumber(vreq); + documentsToReturn = getDocumentsNumber(vreq); } String queryText = getQueryText(vreq); log.debug("Query text is \"" + queryText + "\""); @@ -203,6 +203,8 @@ public static ResponseValues process(VitroRequest vreq, Map SearchResponse response = null; try { + query.setSimpleQuery(!query.getQuery().trim().equals("*:*")); + response = search.query(query); } catch (Exception ex) { String msg = makeBadSearchMessage(queryText, ex.getMessage(), vreq); @@ -259,7 +261,7 @@ public static ResponseValues process(VitroRequest vreq, Map /* Compile the data for the templates */ Map body = new HashMap(); - + /* Add ClassGroup and type refinement links to body */ if (wasHtmlRequested) { if (log.isDebugEnabled()) { @@ -415,7 +417,7 @@ private static int getHitsPerPage(VitroRequest vreq) { log.debug("hitsPerPage is " + hitsPerPage); return hitsPerPage; } - + private static int getDocumentsNumber(VitroRequest vreq) { int documentsNumber = DEFAULT_DOCUMENTS_NUMBER; try { diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/base/BaseSearchQuery.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/base/BaseSearchQuery.java index b6b9018d15..c91057e8f7 100644 --- a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/base/BaseSearchQuery.java +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/base/BaseSearchQuery.java @@ -29,6 +29,7 @@ public class BaseSearchQuery implements SearchQuery { private int facetMinCount = -1; private boolean facetTextToCompareIgnoreCase; private String facetContainsText; + private boolean simpleQuery = false; @Override public SearchQuery setQuery(String query) { @@ -171,4 +172,13 @@ public boolean isFacetTextCompareCaseInsensitive() { return facetTextToCompareIgnoreCase; } + @Override + public void setSimpleQuery(boolean value) { + this.simpleQuery = value; + } + + @Override + public boolean isSimpleQuery() { + return this.simpleQuery; + } } diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/base/SearchEngineUtil.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/base/SearchEngineUtil.java new file mode 100644 index 0000000000..e6c1ca6053 --- /dev/null +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/base/SearchEngineUtil.java @@ -0,0 +1,39 @@ +package edu.cornell.mannlib.vitro.webapp.searchengine.base; + +import java.util.Objects; + +import javax.annotation.Nullable; + +import edu.cornell.mannlib.vitro.webapp.config.ConfigurationProperties; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +public class SearchEngineUtil { + + private static final Log log = LogFactory.getLog(SearchEngineUtil.class); + + @Nullable + public static String getSearchEngineURLProperty() { + ConfigurationProperties config = ConfigurationProperties.getInstance(); + if (Objects.isNull(config)) { + return null; + } + + if (config.getProperty("vitro.local.searchengine.url", "").isEmpty()) { + return tryFetchLegacySolrConfiguration(config); + } + + return config.getProperty("vitro.local.searchengine.url", ""); + } + + private static String tryFetchLegacySolrConfiguration(ConfigurationProperties config) { + String legacyConfigValue = config.getProperty("vitro.local.solr.url", ""); + if (!legacyConfigValue.isEmpty()) { + log.warn( + "vitro.local.solr.url is deprecated, switch to using" + + " vitro.local.searchengine.url as soon as possible."); + } + + return legacyConfigValue; + } +} diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/elasticsearch/CustomQueryBuilder.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/elasticsearch/CustomQueryBuilder.java new file mode 100644 index 0000000000..3398628908 --- /dev/null +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/elasticsearch/CustomQueryBuilder.java @@ -0,0 +1,95 @@ +package edu.cornell.mannlib.vitro.webapp.searchengine.elasticsearch; + +import co.elastic.clients.elasticsearch._types.query_dsl.ExistsQuery; +import co.elastic.clients.elasticsearch._types.query_dsl.FuzzyQuery; +import co.elastic.clients.elasticsearch._types.query_dsl.MatchAllQuery; +import co.elastic.clients.elasticsearch._types.query_dsl.MatchPhraseQuery; +import co.elastic.clients.elasticsearch._types.query_dsl.MatchQuery; +import co.elastic.clients.elasticsearch._types.query_dsl.PrefixQuery; +import co.elastic.clients.elasticsearch._types.query_dsl.Query; +import co.elastic.clients.elasticsearch._types.query_dsl.RangeQuery; +import co.elastic.clients.elasticsearch._types.query_dsl.RegexpQuery; +import co.elastic.clients.elasticsearch._types.query_dsl.WildcardQuery; + +public class CustomQueryBuilder { + + private static final String MAX_FUZZY_EDITS = "2"; + + private CustomQueryBuilder() { + } + + + public static Query buildQuery(SearchType queryType, String field, String value) { + validateInput(field, value); + + switch (queryType) { + case MATCH: + return MatchQuery.of(m -> m + .field(field) + .query(value) + )._toQuery(); + case FUZZY: + return FuzzyQuery.of(m -> m + .field(field) + .value(value.replace("~", "")) + .fuzziness(MAX_FUZZY_EDITS) + )._toQuery(); + case PREFIX: + return PrefixQuery.of(m -> m + .field(field) + .value(value) + )._toQuery(); + case RANGE: + String[] values = value.split("TO"); + return RangeQuery.of(m -> m + .field(field) + .from(values[0].replace("[", "").replace("(", "").trim()) + .to(values[1].replace("]", "").replace(")", "").trim()) + )._toQuery(); + case EXISTS: + return ExistsQuery.of(m -> m + .field(field) + )._toQuery(); + case MATCH_ALL: + return MatchAllQuery.of(m -> m)._toQuery(); + case REGEXP: + String regexpValue; + + boolean isSolrRegexpSpecification = value.startsWith("/") && value.endsWith("/") && value.length() > 1; + if (isSolrRegexpSpecification) { + regexpValue = value.substring(1, value.length() - 1); + } else { + regexpValue = value; + } + + return RegexpQuery.of(m -> m + .field(field) + .value(regexpValue) + )._toQuery(); + case WILDCARD: + if (field.trim().equals("*")) { + return MatchAllQuery.of(m -> m)._toQuery(); + } + + return WildcardQuery.of(m -> m + .field(field) + .value(value.replace(".*", "*")) + )._toQuery(); + default: + return MatchPhraseQuery.of(m -> m + .field(field) + .query(value.length() > 1 ? value.substring(1, value.length() - 1) : value) + // Remove leading and trailing '"' character + )._toQuery(); + } + } + + private static void validateInput(String field, String value) { + if (field == null || field.isEmpty()) { + throw new IllegalArgumentException("Field not specified"); + } + if (value == null) { + throw new IllegalArgumentException("Value not specified"); + } + } +} diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/elasticsearch/ESAdder.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/elasticsearch/ESAdder.java index 860e02b8d8..e688e172ed 100644 --- a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/elasticsearch/ESAdder.java +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/elasticsearch/ESAdder.java @@ -8,18 +8,22 @@ import java.util.HashMap; import java.util.List; import java.util.Map; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.http.client.fluent.Request; -import org.apache.http.client.fluent.Response; -import org.apache.http.entity.ContentType; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import com.fasterxml.jackson.databind.ObjectMapper; - import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngineException; import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchInputDocument; import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchInputField; +import edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames; +import edu.cornell.mannlib.vitro.webapp.utils.http.ESHttpBasicClientFactory; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpPut; +import org.apache.http.entity.StringEntity; +import org.apache.http.util.EntityUtils; /** * The nuts and bolts of adding a document to the Elasticsearch index @@ -44,7 +48,16 @@ private void addDocument(SearchInputDocument doc) throws SearchEngineException { try { Map> map = convertDocToMap(doc); + + if (map.containsKey(VitroSearchTermNames.NAME_RAW)) { + map.putIfAbsent(VitroSearchTermNames.AC_NAME_STEMMED, map.get(VitroSearchTermNames.NAME_RAW)); + map.putIfAbsent(VitroSearchTermNames.AC_NAME_UNTOKENIZED, map.get(VitroSearchTermNames.NAME_RAW)); + } + String json = new ObjectMapper().writeValueAsString(map); + if (json.contains("_drsim")) { + json = reformatDRSIMFields(json); + } log.debug("Adding document for '" + doc.getField("DocId") + "': " + json); @@ -75,15 +88,46 @@ private Map> convertDocToMap(SearchInputDocument doc) { return map; } + private String reformatDRSIMFields(String json) { + String patternString = "\\[(\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}Z) TO (\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}Z)]"; + Pattern pattern = Pattern.compile(patternString); + Matcher matcher = pattern.matcher(json); + + StringBuffer result = new StringBuffer(); + + while (matcher.find()) { + String dateStart = matcher.group(1); + String dateEnd = matcher.group(2); + + String replacement = String.format("{\"gte\": \"%s\", \"lte\": \"%s\"}", dateStart, dateEnd) + .replace("{", "\\{") + .replace("}", "\\}"); + + matcher.appendReplacement(result, replacement); + } + + matcher.appendTail(result); + return result.toString().replace("[\"{", "{").replace("}\"]", "}"); + } + private void putToElastic(String json, String docId) throws SearchEngineException { try { String url = baseUrl + "/_doc/" + URLEncoder.encode(docId, "UTF8"); - Response response = Request.Put(url) - .bodyString(json, ContentType.APPLICATION_JSON).execute(); - log.debug("Response from Elasticsearch: " - + response.returnContent().asString()); + HttpClient httpClient = ESHttpBasicClientFactory.getHttpClient(baseUrl); + + HttpPut request = new HttpPut(url); + request.addHeader("Content-Type", "application/json"); + request.setEntity(new StringEntity(json, "UTF-8")); + HttpResponse response = httpClient.execute(request); + if (response.getStatusLine().getStatusCode() >= 400) { + log.warn("Response from Elasticsearch: " + + EntityUtils.toString(response.getEntity())); + } else { + log.debug("Response from Elasticsearch: " + + EntityUtils.toString(response.getEntity())); + } } catch (Exception e) { throw new SearchEngineException("Failed to put to Elasticsearch", e); diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/elasticsearch/ESCounter.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/elasticsearch/ESCounter.java index 14246484d1..b11eec6d23 100644 --- a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/elasticsearch/ESCounter.java +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/elasticsearch/ESCounter.java @@ -5,12 +5,13 @@ import java.util.HashMap; import java.util.Map; -import org.apache.http.client.fluent.Request; -import org.apache.http.client.fluent.Response; - import com.fasterxml.jackson.databind.ObjectMapper; - import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngineException; +import edu.cornell.mannlib.vitro.webapp.utils.http.ESHttpBasicClientFactory; +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.util.EntityUtils; /** * The nuts and bolts of getting the number of documents in the Elasticsearch @@ -25,9 +26,10 @@ public ESCounter(String baseUrl) { public int count() throws SearchEngineException { try { - String url = baseUrl + "/_doc/_count"; - Response response = Request.Get(url).execute(); - String json = response.returnContent().asString(); + String url = baseUrl + "/_count"; + HttpClient httpClient = ESHttpBasicClientFactory.getHttpClient(baseUrl); + HttpResponse response = httpClient.execute(new HttpGet(url)); + String json = EntityUtils.toString(response.getEntity()); @SuppressWarnings("unchecked") Map map = new ObjectMapper().readValue(json, diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/elasticsearch/ESDeleter.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/elasticsearch/ESDeleter.java index 8267c6f210..ca5a9b3ba9 100644 --- a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/elasticsearch/ESDeleter.java +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/elasticsearch/ESDeleter.java @@ -10,22 +10,24 @@ import java.util.List; import java.util.Map; +import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngineException; +import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchQuery; +import edu.cornell.mannlib.vitro.webapp.searchengine.base.BaseSearchQuery; +import edu.cornell.mannlib.vitro.webapp.utils.http.ESHttpBasicClientFactory; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.http.Header; import org.apache.http.HttpEntity; +import org.apache.http.HttpResponse; import org.apache.http.StatusLine; +import org.apache.http.client.HttpClient; import org.apache.http.client.HttpResponseException; import org.apache.http.client.ResponseHandler; -import org.apache.http.client.fluent.Request; -import org.apache.http.client.fluent.Response; -import org.apache.http.entity.ContentType; +import org.apache.http.client.methods.HttpDelete; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.StringEntity; import org.apache.http.util.EntityUtils; -import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngineException; -import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchQuery; -import edu.cornell.mannlib.vitro.webapp.searchengine.base.BaseSearchQuery; - /** * The nuts and bolts of deleting documents from the Elasticsearch index. */ @@ -52,8 +54,10 @@ private void deleteById(String id) throws SearchEngineException { try { String url = baseUrl + "/_doc/" + URLEncoder.encode(id, "UTF8"); - Response response = Request.Delete(url).execute(); - String json = response.returnContent().asString(); + HttpClient httpClient = ESHttpBasicClientFactory.getHttpClient(baseUrl); + + HttpResponse response = httpClient.execute(new HttpDelete(url)); + String json = EntityUtils.toString(response.getEntity()); } catch (HttpResponseException e) { if (e.getStatusCode() == 404) { // Don't care if it has already been deleted. @@ -69,16 +73,23 @@ private void deleteById(String id) throws SearchEngineException { public void deleteByQuery(String queryString) throws SearchEngineException { String url = baseUrl + "/_delete_by_query"; + queryString = queryString.replace(" ", ""); + if (queryString.contains("*TO")) { + queryString = queryString.replace("[", "").replace("]", "").replace("*", "0"); + } SearchQuery query = new BaseSearchQuery().setQuery(queryString); - String queryJson = new QueryConverter(query).asString(); + String queryJson = new QueryConverter(query, true).asString(); try { - Response response = Request.Post(url) - .bodyString(queryJson, ContentType.APPLICATION_JSON) - .execute(); + HttpClient httpClient = ESHttpBasicClientFactory.getHttpClient(baseUrl); + + HttpPost request = new HttpPost(url); + request.addHeader("Content-Type", "application/json"); + request.setEntity(new StringEntity(queryJson)); + HttpResponse response = httpClient.execute(request); BaseResponseHandler handler = new BaseResponseHandler(); - response.handleResponse(handler); + handler.handleResponse(response); if (handler.getStatusCode() >= 400) { log.warn(String.format( "Failed to delete Elasticsearch documents by query: %s, %d - %s\n%s", diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/elasticsearch/ESFlusher.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/elasticsearch/ESFlusher.java index 1b32e8e455..223ce5a001 100644 --- a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/elasticsearch/ESFlusher.java +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/elasticsearch/ESFlusher.java @@ -2,12 +2,14 @@ package edu.cornell.mannlib.vitro.webapp.searchengine.elasticsearch; +import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngineException; +import edu.cornell.mannlib.vitro.webapp.utils.http.ESHttpBasicClientFactory; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.http.client.fluent.Request; -import org.apache.http.client.fluent.Response; - -import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngineException; +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.util.EntityUtils; /** * Just does a "commit" or "flush" to the index. @@ -29,8 +31,9 @@ public void flush(boolean wait) throws SearchEngineException { try { String url = baseUrl + "/_flush" + (wait ? "?wait_for_ongoing" : ""); - Response response = Request.Get(url).execute(); - String json = response.returnContent().asString(); + HttpClient httpClient = ESHttpBasicClientFactory.getHttpClient(baseUrl); + HttpResponse response = httpClient.execute(new HttpGet(url)); + String json = EntityUtils.toString(response.getEntity()); log.debug("flush response: " + json); } catch (Exception e) { throw new SearchEngineException("Failed to put to Elasticsearch", diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/elasticsearch/ESQuery.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/elasticsearch/ESQuery.java index 01fcf37f39..7d12391d99 100644 --- a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/elasticsearch/ESQuery.java +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/elasticsearch/ESQuery.java @@ -6,6 +6,8 @@ import java.net.URI; import java.net.URISyntaxException; +import edu.cornell.mannlib.vitro.webapp.utils.http.HttpClientFactory; +import edu.cornell.mannlib.vitro.webapp.utils.http.ESHttpBasicClientFactory; import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -17,7 +19,6 @@ import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngineException; import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchQuery; import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResponse; -import edu.cornell.mannlib.vitro.webapp.utils.http.HttpClientFactory; /** * Convert a SearchQuery to JSON, send it to Elasticsearch, and convert the JSON @@ -34,9 +35,12 @@ public ESQuery(String baseUrl) { public SearchResponse query(SearchQuery query) throws SearchEngineException { - String queryString = new QueryConverter(query).asString(); + boolean treatAsStructuredQuery = !query.isSimpleQuery(); + + String queryString = new QueryConverter(query, treatAsStructuredQuery).asString(); String response = doTheQuery(queryString); - return new ResponseParser(response).parse(); + return new ResponseParser(response) + .parse(query.getFacetTextToMatch(), query.isFacetTextCompareCaseInsensitive()); } private String doTheQuery(String queryString) { @@ -65,10 +69,10 @@ private String doTheQuery(String queryString) { * allow you to put a body on a GET request. In online discussion, some say * that the HTTP spec is ambiguous on this point, so each implementation * makes its own choice. For example, CURL allows it. - * + * * More to the point however, is that ElasticSearch requires it. So here's a * simple class to make that possible. - * + * * USE POST INSTEAD!! */ private static class ESFunkyGetRequest @@ -90,7 +94,10 @@ public ESFunkyGetRequest bodyString(String contents, public HttpResponse execute() throws SearchEngineException { try { - return HttpClientFactory.getHttpClient().execute(this); + if (this.getURI().getScheme().equals("https")) { + return ESHttpBasicClientFactory.getHttpsClient().execute(this); + } + return ESHttpBasicClientFactory.getHttpClient().execute(this); } catch (IOException e) { throw new SearchEngineException(e); } diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/elasticsearch/ElasticSearchEngine.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/elasticsearch/ElasticSearchEngine.java index 80b1da984a..8442614fb3 100644 --- a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/elasticsearch/ElasticSearchEngine.java +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/elasticsearch/ElasticSearchEngine.java @@ -2,12 +2,11 @@ package edu.cornell.mannlib.vitro.webapp.searchengine.elasticsearch; +import java.io.IOException; import java.util.Arrays; import java.util.Collection; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - +import edu.cornell.mannlib.vitro.webapp.config.ConfigurationProperties; import edu.cornell.mannlib.vitro.webapp.modules.Application; import edu.cornell.mannlib.vitro.webapp.modules.ComponentStartupStatus; import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngine; @@ -17,11 +16,15 @@ import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResponse; import edu.cornell.mannlib.vitro.webapp.searchengine.base.BaseSearchInputDocument; import edu.cornell.mannlib.vitro.webapp.searchengine.base.BaseSearchQuery; -import edu.cornell.mannlib.vitro.webapp.utils.configuration.Property; -import edu.cornell.mannlib.vitro.webapp.utils.configuration.Validation; +import edu.cornell.mannlib.vitro.webapp.utils.http.ESHttpBasicClientFactory; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpHead; /** - * A first draft of an Elasticsearch implementation. + * A first version of an Elasticsearch engine implementation. */ public class ElasticSearchEngine implements SearchEngine { private static final Log log = LogFactory.getLog(ElasticSearchEngine.class); @@ -32,47 +35,47 @@ public class ElasticSearchEngine implements SearchEngine { private String baseUrl; - @Property(uri = "http://vitro.mannlib.cornell.edu/ns/vitro/ApplicationSetup#hasBaseUrl") - public void setBaseUrl(String url) { - if (baseUrl == null) { - if (url.endsWith("/")) { - url = url.substring(0, url.length() - 1); - } - baseUrl = url; - } else { - throw new IllegalStateException( - "Configuration includes multiple base URLs: " + url - + ", and " + baseUrl); - } - } - - @Validation - public void validate() throws Exception { - if (baseUrl == null) { - throw new IllegalStateException( - "Configuration did not include a base URL."); - } - } - // ---------------------------------------------------------------------- // The instance // ---------------------------------------------------------------------- @Override - public void startup(Application application, ComponentStartupStatus ss) { - log.warn("ElasticSearchEngine.startup() not implemented."); // TODO + public void startup(Application application, ComponentStartupStatus css) { + String elasticUrlString = ConfigurationProperties.getInstance().getProperty("vitro.local.searchengine.url", ""); + if (elasticUrlString.isEmpty()) { + css.fatal("Can't connect to ElasticSearch engine. " + + "runtime.properties must contain a value for " + + "vitro.local.searchengine.url"); + } + + baseUrl = elasticUrlString; } @Override public void shutdown(Application application) { - // TODO Flush the buffers - log.warn("ElasticSearchEngine.shutdown not implemented."); + try { + new ESFlusher(baseUrl).flush(true); + } catch (SearchEngineException e) { + log.warn("Unexpected error upon Elasticsearch engine shutdown. A component has thrown an error: " + + e.getMessage()); + } } @Override public void ping() throws SearchEngineException { - // TODO What's the simplest we can do? Another smoke test? - log.warn("ElasticSearchEngine.ping() not implemented."); // TODO + HttpHead httpHead = new HttpHead(baseUrl); + HttpClient httpClient = ESHttpBasicClientFactory.getHttpClient(baseUrl); + + try { + HttpResponse response = httpClient.execute(httpHead); + int statusCode = response.getStatusLine().getStatusCode(); + if (statusCode != 200) { + throw new SearchEngineException( + "Failed to ping Elasticsearch - ES responded with status code " + statusCode); + } + } catch (SearchEngineException | IOException e) { + throw new SearchEngineException("Failed to put to Elasticsearch - request failed"); + } } @Override @@ -87,7 +90,7 @@ public void add(SearchInputDocument... docs) throws SearchEngineException { @Override public void add(Collection docs) - throws SearchEngineException { + throws SearchEngineException { new ESAdder(baseUrl).add(docs); } @@ -108,7 +111,7 @@ public void deleteById(String... ids) throws SearchEngineException { @Override public void deleteById(Collection ids) - throws SearchEngineException { + throws SearchEngineException { new ESDeleter(baseUrl).deleteByIds(ids); } @@ -131,7 +134,7 @@ public SearchQuery createQuery(String queryText) { @Override public SearchResponse query(SearchQuery query) - throws SearchEngineException { + throws SearchEngineException { return new ESQuery(baseUrl).query(query); } diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/elasticsearch/ElasticSearchResultDocumentList.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/elasticsearch/ElasticSearchResultDocumentList.java index a10e4670b5..1bca1187b5 100644 --- a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/elasticsearch/ElasticSearchResultDocumentList.java +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/elasticsearch/ElasticSearchResultDocumentList.java @@ -30,7 +30,7 @@ public Iterator iterator() { @Override public long getNumFound() { - return documents.size(); + return numberFound; } @Override diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/elasticsearch/Elasticsearch_notes_on_the_first_draft.md b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/elasticsearch/Elasticsearch_notes_on_the_first_draft.md index 2ffd3aade5..b5ab3f9f6b 100644 --- a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/elasticsearch/Elasticsearch_notes_on_the_first_draft.md +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/elasticsearch/Elasticsearch_notes_on_the_first_draft.md @@ -11,6 +11,7 @@ * Create a search index with the appropriate mapping (see below). * Check out VIVO and this branch of Vitro (see below), and do the usual installation procedure. * Modify `{vitro_home}/config/applicationSetup.n3` to use this driver (see below). +* Modify the `vitro.local.searchengine.url` configuration property to contain ES index base URL * Start elasticsearch * Start VIVO @@ -65,57 +66,192 @@ git clone -b feature/elasticsearchExperiments https://github.com/j2blake/Vitro.g ``` curl -X PUT "localhost:9200/vivo?pretty" -H 'Content-Type: application/json' -d' { - "mappings": { - "_doc": { - "properties": { - "ALLTEXT": { - "type": "text", - "analyzer": "english" - }, - "ALLTEXTUNSTEMMED": { - "type": "text", - "analyzer": "standard" - }, - "DocId": { - "type": "keyword" - }, - "classgroup": { - "type": "keyword" - }, - "type": { - "type": "keyword" - }, - "mostSpecificTypeURIs": { - "type": "keyword" - }, - "indexedTime": { - "type": "long" + "settings":{ + "index":{ + "analysis":{ + "tokenizer":{ + "keyword_tokenizer":{ + "type":"keyword" + }, + "whitespace_tokenizer":{ + "type":"whitespace" + } }, - "nameRaw": { - "type": "keyword" - }, - "URI": { - "type": "keyword" - }, - "THUMBNAIL": { - "type": "integer" - }, - "THUMBNAIL_URL": { - "type": "keyword" - }, - "nameLowercaseSingleValued": { - "type": "text", - "analyzer": "standard", - "fielddata": "true" + "filter":{ + "lowercase_filter":{ + "type":"lowercase" + }, + "edgengram_filter":{ + "type":"edge_ngram", + "min_gram":2, + "max_gram":25 + }, + "word_delimiter_filter":{ + "type":"word_delimiter", + "generate_word_parts":true, + "generate_number_parts":true, + "catenate_words":false, + "catenate_numbers":false, + "catenate_all":false, + "split_on_case_change":true + }, + "porter_stem_filter":{ + "type":"snowball", + "language":"English" + } }, - "BETA" : { - "type" : "float" + "analyzer":{ + "default":{ + "type":"english" + }, + "edgengram_untokenized":{ + "type":"custom", + "tokenizer":"keyword_tokenizer", + "filter":[ + "lowercase_filter", + "edgengram_filter" + ] + }, + "edgengram_untokenized_query":{ + "type":"custom", + "tokenizer":"keyword_tokenizer", + "filter":[ + "lowercase_filter" + ] + }, + "edgengram_stemmed":{ + "type":"custom", + "tokenizer":"whitespace_tokenizer", + "filter":[ + "word_delimiter_filter", + "lowercase_filter", + "porter_stem_filter", + "edgengram_filter" + ] + }, + "edgengram_stemmed_query":{ + "type":"custom", + "tokenizer":"whitespace_tokenizer", + "filter":[ + "word_delimiter_filter", + "lowercase_filter", + "porter_stem_filter" + ] + }, + "sort_field_analyzer":{ + "type":"custom", + "tokenizer":"keyword", + "filter":[ + "lowercase" + ] + } } } } }, - "query": { - "default_field": "ALLTEXT" + "mappings":{ + "dynamic_templates":[ + { + "field_sort_template":{ + "match":"*_label_sort", + "mapping":{ + "type":"text", + "fields":{ + "keyword":{ + "type":"keyword" + } + }, + "fielddata":true, + "analyzer":"sort_field_analyzer" + } + } + }, + { + "field_ss_template":{ + "match":"*_ss", + "mapping":{ + "type":"text", + "fields":{ + "keyword":{ + "type":"keyword", + "ignore_above":256 + } + }, + "fielddata":true + } + } + }, + { + "date_range_template":{ + "match":"*_drsim", + "mapping":{ + "type":"date_range", + "format":"strict_date_optional_time||epoch_millis" + } + } + } + ], + "properties":{ + "ALLTEXT":{ + "type":"text", + "analyzer":"english", + "fields":{ + "keyword":{ + "type":"keyword", + "ignore_above":256 + } + } + }, + "ALLTEXTUNSTEMMED":{ + "type":"text", + "analyzer":"standard" + }, + "DocId":{ + "type":"keyword" + }, + "classgroup":{ + "type":"keyword" + }, + "type":{ + "type":"keyword" + }, + "mostSpecificTypeURIs":{ + "type":"keyword" + }, + "indexedTime":{ + "type":"long" + }, + "nameRaw":{ + "type":"keyword" + }, + "URI":{ + "type":"keyword" + }, + "THUMBNAIL":{ + "type":"integer" + }, + "THUMBNAIL_URL":{ + "type":"keyword" + }, + "nameLowercaseSingleValued":{ + "type":"text", + "analyzer":"standard", + "fielddata":true + }, + "BETA":{ + "type":"float" + }, + "acNameUntokenized":{ + "type":"text", + "analyzer":"edgengram_untokenized", + "search_analyzer":"edgengram_untokenized_query" + }, + "acNameStemmed":{ + "type":"text", + "analyzer":"edgengram_stemmed", + "search_analyzer":"edgengram_stemmed_query" + } + } } } ' @@ -165,6 +301,7 @@ Again, any location and port may be used, but they must match the "base URL" in ; :hasBaseUrl "http://localhost:9200/vivo" . ``` +Note that `hasBaseUrl "http://localhost:9200/vivo" .` can be omitted. ## Enhance the contents of the search index ### An example: Publication URIs in the author's search document diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/elasticsearch/ExpressionTransformer.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/elasticsearch/ExpressionTransformer.java new file mode 100644 index 0000000000..0987be34a3 --- /dev/null +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/elasticsearch/ExpressionTransformer.java @@ -0,0 +1,352 @@ +package edu.cornell.mannlib.vitro.webapp.searchengine.elasticsearch; + +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Deque; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.Stack; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import co.elastic.clients.elasticsearch._types.query_dsl.BoolQuery; +import co.elastic.clients.elasticsearch._types.query_dsl.Query; + +public class ExpressionTransformer { + + private static final Map priorities; + + static { + Map priorityTempMap = new HashMap<>(); + priorityTempMap.put("AND", 2); + priorityTempMap.put("OR", 1); + priorityTempMap.put("NOT", 3); + priorityTempMap.put("(", 0); + priorities = Collections.unmodifiableMap(priorityTempMap); + } + + public static String removeWhitespacesFromRangeExpression(String expression) { + String regex = "\\[\\s*([^\\s]+)\\s+TO\\s+([^\\s]+)\\s*\\]"; + Pattern pattern = Pattern.compile(regex); + Matcher matcher = pattern.matcher(expression); + + StringBuffer result = new StringBuffer(); + while (matcher.find()) { + String replacement = "[" + matcher.group(1) + "TO" + matcher.group(2) + "]"; + matcher.appendReplacement(result, replacement); + } + matcher.appendTail(result); + + return result.toString(); + } + + public static String fillInMissingOperators(String query) { + String[] tokens = removeInvalidParentheses(List.of(query.split(" "))).toArray(new String[0]); + StringBuilder modifiedQuery = new StringBuilder(); + + boolean insidePhrase = false; + int parenBalance = 0; + String currentToken = ""; + + for (int i = 0; i < tokens.length; i++) { + if (tokens[i].trim().isEmpty()) { + continue; + } + + currentToken = insidePhrase ? currentToken + " " + tokens[i] : tokens[i]; + + if (currentToken.startsWith("\"") && !insidePhrase && hasClosingQuote(tokens, i)) { + insidePhrase = true; + } else if (currentToken.endsWith("\"") && insidePhrase) { + insidePhrase = false; + } + + boolean isOpeningParen = currentToken.equals("("); + boolean isClosingParen = currentToken.equals(")"); + boolean isLogicalOperator = priorities.containsKey(currentToken); + boolean isFieldQuery = isTokenAPredefinedFieldQuery(currentToken); + + if (isFieldQuery && currentToken.contains(":\"") && !currentToken.endsWith("\"")) { + modifiedQuery.append(currentToken.split(":")[0]).append(":"); + currentToken = currentToken.split(":")[1]; + insidePhrase = true; + } + + if (isOpeningParen) { + parenBalance++; + } else if (isClosingParen) { + if (parenBalance == 0) { + continue; + } + parenBalance--; + } + + if (i > 0 && isOpeningParen && !priorities.containsKey(tokens[i - 1]) && !tokens[i - 1].equals("(") && + hasClosingParen(tokens, i)) { + modifiedQuery.append(" AND "); + } + + if (i > 0 && tokens[i - 1].equals(")") && !insidePhrase && !isLogicalOperator && !isClosingParen) { + modifiedQuery.append(" AND "); + } + + if (i > 0 && isFieldQuery && !priorities.containsKey(tokens[i - 1]) && !tokens[i - 1].equals("(")) { + modifiedQuery.append(" OR "); + } + + if (!isLogicalOperator && !isFieldQuery && !isOpeningParen && !isClosingParen && !insidePhrase) { + if (i > 0 && modifiedQuery.toString().trim().endsWith("(")) { + modifiedQuery.append(" OR "); + } else if (!modifiedQuery.toString().endsWith(":")) { + currentToken = "( ALLTEXT:" + currentToken + " OR nameLowercaseSingleValued:" + currentToken + " )"; + } + } + + if (!insidePhrase) { + modifiedQuery.append(currentToken); + if (i < tokens.length - 1) { + modifiedQuery.append(" "); + } + } + } + + if (modifiedQuery.length() == 0 && insidePhrase) { + return "ALLTEXT: " + currentToken + "\""; + } + + return modifiedQuery.toString(); + } + + public static List removeInvalidParentheses(List tokens) { + List result = new ArrayList<>(); + Deque openParenIndexes = new ArrayDeque<>(); + + boolean insideQuotes = false; + + Set invalidIndexes = new HashSet<>(); + + for (int i = 0; i < tokens.size(); i++) { + String token = tokens.get(i); + + long quoteCount = token.chars().filter(ch -> ch == '"').count(); + if (quoteCount % 2 != 0) { + insideQuotes = !insideQuotes; + } + + if (!insideQuotes) { + if (token.equals("(")) { + openParenIndexes.push(i); + } else if (token.equals(")")) { + if (!openParenIndexes.isEmpty()) { + openParenIndexes.pop(); // matched + } else { + invalidIndexes.add(i); // unmatched closing paren + } + } + } + } + + invalidIndexes.addAll(openParenIndexes); + + for (int i = 0; i < tokens.size(); i++) { + if (!invalidIndexes.contains(i) && !tokens.get(i).isBlank()) { + result.add(tokens.get(i)); + } + } + + return result; + } + + private static boolean hasClosingParen(String[] tokens, int fromIndex) { + boolean insideQuotes = false; + + for (int j = fromIndex + 1; j < tokens.length; j++) { + String token = tokens[j]; + + long quoteCount = token.chars().filter(ch -> ch == '"').count(); + if (quoteCount % 2 != 0) { + insideQuotes = !insideQuotes; + } + + if (!insideQuotes && token.equals(")")) { + return true; + } + } + + return false; + } + + private static boolean hasClosingQuote(String[] tokens, int fromIndex) { + boolean insideQuotes = false; + for (int j = fromIndex; j < tokens.length; j++) { + long quoteCount = tokens[j].chars().filter(ch -> ch == '"').count(); + if (quoteCount % 2 != 0) { + insideQuotes = !insideQuotes; + } + } + return !insideQuotes; + } + + public static boolean isTokenAPredefinedFieldQuery(String token) { + if (!token.contains(":")) { + return false; + } + + return !token.startsWith(":") && !token.endsWith(":"); + } + + private static SearchType decideQueryType(String field, String value) { + SearchType searchType = SearchType.MATCH; + + if (value.startsWith("\"") && value.endsWith("\"")) { + searchType = SearchType.PHRASE; + } else if (value.contains("TO") && !value.equals("TO")) { + if (value.replace(" ", "").equals("[*TO*]")) { + searchType = SearchType.EXISTS; + } else { + searchType = SearchType.RANGE; + } + } else if (field.contains("*") || value.contains("*")) { + if (field.trim().equals("*")) { + searchType = SearchType.MATCH_ALL; + } else if (value.startsWith("/") && value.endsWith("/") && value.length() > 1) { + searchType = SearchType.REGEXP; + } else { + searchType = SearchType.WILDCARD; + } + } else if (value.endsWith("~")) { + searchType = SearchType.FUZZY; + } + + return searchType; + } + + public Query parseAdvancedQuery(List expression) { + return buildQueryFromPostFixExpression(transformToPostFixNotation(expression)); + } + + private List transformToPostFixNotation(List expression) { + Stack tokenStack = new Stack<>(); + ArrayList postfixExpression = new ArrayList<>(); + + List fixedTokens = fixDisjointFieldTokens(expression); + + for (String token : fixedTokens) { + if (!priorities.containsKey(token) && !token.equals(")")) { + postfixExpression.add(token); + } else if (token.equals("(")) { + tokenStack.push(token); + } else if (token.equals(")")) { + while (!tokenStack.isEmpty() && !tokenStack.peek().equals("(")) { + postfixExpression.add(tokenStack.pop()); + } + tokenStack.pop(); // Remove the '(' from stack + } else { + while (!tokenStack.isEmpty() && + priorities.get(token) <= priorities.getOrDefault(tokenStack.peek(), 0)) { + postfixExpression.add(tokenStack.pop()); + } + tokenStack.push(token); + } + } + + while (!tokenStack.isEmpty()) { + postfixExpression.add(tokenStack.pop()); + } + + return postfixExpression; + } + + private List fixDisjointFieldTokens(List expression) { + List fixedTokens = new ArrayList<>(); + Set specialCharacters = Set.of("AND", "OR", "NOT", "(", ")"); + + for (int i = 0; i < expression.size(); i++) { + String token = expression.get(i); + + if ("TO".equals(token) && i > 0 && i < expression.size() - 1) { + String before = fixedTokens.remove(fixedTokens.size() - 1); + String after = expression.get(i + 1); + fixedTokens.add((before + " TO " + after).replace("\"", "")); + i++; + continue; + } + + if (!fixedTokens.isEmpty()) { + String prev = fixedTokens.get(fixedTokens.size() - 1); + + if (prev.contains(":") && + !token.contains(":") && + !specialCharacters.contains(token) && + !"TO".equals(token)) { + + fixedTokens.set(fixedTokens.size() - 1, prev + " " + token); + continue; + } + } + + fixedTokens.add(token); + } + + return fixedTokens; + } + + private Query buildQueryFromPostFixExpression(List postfixExpression) { + Stack queryStack = new Stack<>(); + + for (String token : postfixExpression) { + switch (token.toUpperCase()) { + case "AND": + Query mustContain = queryStack.pop(); + queryStack.push(BoolQuery.of(q -> { + q.must(mustContain); + q.must(queryStack.pop()); + return q; + })._toQuery()); + break; + case "OR": + Query shouldContain = queryStack.pop(); + queryStack.push(BoolQuery.of(q -> { + q.should(shouldContain); + q.should(queryStack.pop()); + return q; + })._toQuery()); + break; + case "NOT": + Query mustNotContain = queryStack.pop(); + queryStack.push(BoolQuery.of(q -> { + q.must(queryStack.pop()); + q.mustNot(mustNotContain); + return q; + })._toQuery()); + break; + default: + String[] fieldValueTuple = token.split(":", 2); + + if (fieldValueTuple[0].startsWith("-")) { + queryStack.push(BoolQuery.of(q -> { + q.mustNot(CustomQueryBuilder.buildQuery( + decideQueryType(fieldValueTuple[0], fieldValueTuple[1]), + fieldValueTuple[0].replaceFirst("-", ""), + fieldValueTuple[1])); + return q; + })._toQuery()); + break; + } + + SearchType searchType = decideQueryType(fieldValueTuple[0], fieldValueTuple[1]); + + queryStack.push(CustomQueryBuilder.buildQuery( + searchType, + fieldValueTuple[0], + fieldValueTuple[1])); + } + } + + return queryStack.pop(); + } +} diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/elasticsearch/QueryConverter.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/elasticsearch/QueryConverter.java index d756054b9e..acccfb3745 100644 --- a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/elasticsearch/QueryConverter.java +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/elasticsearch/QueryConverter.java @@ -7,19 +7,21 @@ import static edu.cornell.mannlib.vitro.webapp.searchengine.elasticsearch.JsonTree.tree; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - +import co.elastic.clients.elasticsearch._types.query_dsl.BoolQuery; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; - import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngineException; import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchQuery; import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchQuery.Order; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; /** * Accept a SearchQuery and make it available as a JSON string, suitable for @@ -36,9 +38,9 @@ public class QueryConverter { private final List returnFields; private final Map fullMap; - public QueryConverter(SearchQuery query) { + public QueryConverter(SearchQuery query, boolean treatAsLuceneQuery) { this.query = query; - this.queryAndFilters = filteredOrNot(); + this.queryAndFilters = filteredOrNot(treatAsLuceneQuery); this.sortFields = figureSortFields(); this.facets = figureFacets(); this.highlighter = figureHighlighter(); @@ -47,20 +49,66 @@ public QueryConverter(SearchQuery query) { this.fullMap = figureFullMap(); } - private Map filteredOrNot() { - if (query.getFilters().isEmpty()) { - return new QueryStringMap(query.getQuery()).map; - } else { - return buildFilterStructure(); + private Map filteredOrNot(boolean treatAsStructuredQuery) { + ObjectMapper objectMapper = new ObjectMapper(); + try { + return objectMapper.readValue(getES8Query(treatAsStructuredQuery).replaceFirst("Query: ", ""), + new TypeReference>() { + }); + } catch (JsonProcessingException e) { + log.error("Query parsing for ES8 failed, parsing it as unstructured query."); + if (treatAsStructuredQuery) { + return filteredOrNot(false); + } } + + log.error("Query parsing for ES8 failed, falling back to old parsing method for query " + query.getQuery()); + return new QueryStringMap(query.getQuery()).map; } private Map buildFilterStructure() { return tree() // - .put("bool", tree() // - .put("must", new QueryStringMap(query.getQuery()).map) // - .put("filter", buildFiltersList())) // - .asMap(); + .put("bool", tree() // + .put("must", new QueryStringMap(query.getQuery()).map) // + .put("filter", buildFiltersList())) // + .asMap(); + } + + private String getES8Query(Boolean treatAsStructuredQuery) { + ExpressionTransformer transformer = new ExpressionTransformer(); + + StringBuilder queryForParsing; + if (treatAsStructuredQuery) { + queryForParsing = new StringBuilder("( " + ExpressionTransformer.fillInMissingOperators( + ExpressionTransformer.removeWhitespacesFromRangeExpression( + query.getQuery() + .replace("(", "( ") + .replace(")", " )") + )) + " )"); + } else { + queryForParsing = new StringBuilder("( " + Arrays.stream(query.getQuery().trim().split("\\s+")) + .filter(token -> !token.isBlank() && !token.equals("\"")) + .map(token -> { + if (token.startsWith("classgroup:")) { + return token; + } + + return "ALLTEXT:" + token + " OR nameLowercaseSingleValued:" + token; + }) + .collect(Collectors.joining(" OR ")) + " )"); + } + + for (String filter : query.getFilters()) { + queryForParsing.append(" AND ").append(filter); + } + + List queryTokens = new ArrayList<>(Arrays.asList(queryForParsing.toString().split(" "))); + queryTokens.removeIf(String::isEmpty); + + return BoolQuery.of(q -> { + q.must(transformer.parseAdvancedQuery(queryTokens)); + return q; + })._toQuery().toString(); } private List> buildFiltersList() { @@ -76,7 +124,11 @@ private Map figureSortFields() { Map map = new HashMap<>(); for (String name : fields.keySet()) { String sortOrder = fields.get(name).toString().toLowerCase(); - map.put(name, sortOrder); + if (name.equals("score")) { + map.put("_score", sortOrder); + } else { + map.put(name, sortOrder); + } } return map; } @@ -91,19 +143,31 @@ private Map figureFacets() { private Map figureHighlighter() { return tree() // - .put("fields", tree() // - .put("ALLTEXT", EMPTY_JSON_MAP)) - .asMap(); + .put("fields", tree() // + .put("ALLTEXT", EMPTY_JSON_MAP)) + .asMap(); } private Map figureFacet(String field) { - return tree() // - .put("terms", tree() // - .put("field", field) // - .put("size", ifPositive(query.getFacetLimit())) // - .put("min_doc_count", - ifPositive(query.getFacetMinCount()))) // - .asMap(); + String fieldToAggregate = field.endsWith("_ss") ? field + ".keyword" : field; + + return tree() + .put("terms", tree() + .put("field", fieldToAggregate) + .put("size", ifPositive(query.getFacetLimit())) + .put("min_doc_count", ifPositive(query.getFacetMinCount())) + ) + .put("aggs", tree() + .put("top_label", tree() + .put("top_hits", tree() + .put("size", 1) + .put("_source", + List.of("es_label_display") // fetch default display label from index source + ) + ) + ) + ) + .asMap(); } private List figureReturnFields() { @@ -112,14 +176,15 @@ private List figureReturnFields() { private Map figureFullMap() { return tree() // - .put("query", queryAndFilters) // - .put("from", ifPositive(query.getStart())) // - .put("highlight", highlighter) - .put("size", ifPositive(query.getRows())) // - .put("sort", sortFields) // - .put("_source", returnFields) // - .put("aggregations", facets) // - .asMap(); + .put("track_total_hits", true) + .put("query", queryAndFilters) // + .put("from", ifPositive(query.getStart())) // + .put("highlight", highlighter) + .put("size", ifPositive(query.getRows())) // + .put("sort", sortFields) // + .put("_source", returnFields) // + .put("aggs", facets) // + .asMap(); } public String asString() throws SearchEngineException { @@ -140,25 +205,25 @@ public QueryStringMap(String queryString) { /** * This is a kluge, but perhaps it will work for now. - * + *

* Apparently Solr is willing to put up with query strings that contain * special characters in odd places, but Elasticsearch is not. - * + *

* So, a query string of "classgroup:http://this/that" must be escaped * as "classgroup:http\:\/\/this\/that". Notice that the first colon * delimits the field name, and so must not be escaped. - * + *

* But what if no field is specified? Then all colons must be escaped. * How would we distinguish that? - * + *

* And what if the query is more complex, and more than one field is * specified? What if other special characters are included? - * + *

* This could be a real problem. */ private String escape(String queryString) { return queryString.replace(":", "\\:").replace("/", "\\/") - .replaceFirst("\\\\:", ":"); + .replaceFirst("\\\\:", ":"); } private Map makeInnerMap(String queryString) { diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/elasticsearch/ResponseParser.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/elasticsearch/ResponseParser.java index 7ac521065a..731c47eb9d 100644 --- a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/elasticsearch/ResponseParser.java +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/elasticsearch/ResponseParser.java @@ -5,15 +5,13 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - +import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; - import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngineException; import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchFacetField; import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchFacetField.Count; @@ -23,6 +21,8 @@ import edu.cornell.mannlib.vitro.webapp.searchengine.base.BaseSearchFacetField.BaseCount; import edu.cornell.mannlib.vitro.webapp.searchengine.base.BaseSearchResponse; import edu.cornell.mannlib.vitro.webapp.searchengine.base.BaseSearchResultDocument; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; /** * Elastic search sends a JSON response to a query. parse it to a @@ -41,29 +41,29 @@ class ResponseParser { @SuppressWarnings("unchecked") public ResponseParser(String responseString) throws SearchEngineException { try { - this.responseMap = new ObjectMapper().readValue(responseString, - HashMap.class); + this.responseMap = new ObjectMapper().readValue(responseString, new TypeReference>() { + }); } catch (IOException e) { throw new SearchEngineException(e); } } - public SearchResponse parse() { + public SearchResponse parse(String facetTextToMatch, boolean isFacetTextCompareCaseInsensitive) { parseDocumentList(); - parseFacetFields(); + parseFacetFields(facetTextToMatch, isFacetTextCompareCaseInsensitive); SearchResponse response = new BaseSearchResponse(highlightingMap, - facetFieldsMap, - new ElasticSearchResultDocumentList(documentList, totalHits)); + facetFieldsMap, + new ElasticSearchResultDocumentList(documentList, totalHits)); log.debug("ESQuery.ResponseParser.parse: " + response); return response; } - private void parseFacetFields() { + private void parseFacetFields(String facetTextToMatch, boolean isFacetTextCompareCaseInsensitive) { facetFieldsMap = new HashMap<>(); @SuppressWarnings("unchecked") Map> aggregations = (Map>) responseMap - .get("aggregations"); + .get("aggregations"); if (aggregations == null) { return; } @@ -71,25 +71,62 @@ private void parseFacetFields() { for (String key : aggregations.keySet()) { if (key.startsWith("facet_")) { String name = key.substring(6); - parseFacetField(name, aggregations.get(key)); + parseFacetField(name, aggregations.get(key), facetTextToMatch, isFacetTextCompareCaseInsensitive); } } } - private void parseFacetField(String name, Map facetMap) { - @SuppressWarnings("unchecked") - List> bucketsList = (List>) facetMap - .get("buckets"); + @SuppressWarnings("unchecked") + private void parseFacetField(String name, Map facetMap, String facetText, boolean ignoreCase) { + List> bucketsList = (List>) facetMap.get("buckets"); if (bucketsList == null) { return; } List counts = new ArrayList<>(); for (Map bucket : bucketsList) { - counts.add(new BaseCount((String) bucket.get("key"), - (Integer) bucket.get("doc_count"))); - } + String key = (String) bucket.get("key"); + int count = (Integer) bucket.get("doc_count"); + + String label = key; // default value + + // Not needed initially, but useful as a POC for enhancing implementation + // if we want to support this functionality with URI-key aggregations + Map topLabel = (Map) bucket.get("top_label"); + if (topLabel != null && key.contains("/individual/")) { + Map hits = (Map) topLabel.get("hits"); + if (hits != null) { + List> hitList = (List>) hits.get("hits"); + if (hitList != null && !hitList.isEmpty()) { + Map firstHit = hitList.get(0); + Map source = (Map) firstHit.get("_source"); + if (source != null) { + for (Map.Entry entry : source.entrySet()) { + String fieldName = entry.getKey(); + if (fieldName.endsWith("es_label_display") && entry.getValue() instanceof List) { + List labels = (List) entry.getValue(); + if (!labels.isEmpty()) { + label = String.valueOf(labels.get(0)); + break; + } + } + } + } + } + } + } + if (facetText != null) { + boolean matches = ignoreCase + ? label.toLowerCase().contains(facetText.toLowerCase()) + : label.contains(facetText); + if (!matches) { + continue; + } + } + + counts.add(new BaseCount(key, count)); + } facetFieldsMap.put(name, new BaseSearchFacetField(name, counts)); } @@ -98,33 +135,38 @@ private void parseDocumentList() { highlightingMap = new HashMap<>(); @SuppressWarnings("unchecked") - Map uberHits = (Map) responseMap - .get("hits"); + Map uberHits = (Map) responseMap.get("hits"); if (uberHits == null) { - log.warn("Didn't find a 'hits' field " + "in the query response: " - + responseMap); + log.warn("Didn't find a 'hits' field in the query response: " + responseMap); return; } - Integer total = (Integer) uberHits.get("total"); + // Updated handling of the 'total' field + @SuppressWarnings("unchecked") + Map totalMap = (Map) uberHits.get("total"); + if (totalMap == null) { + log.warn("Didn't find a 'hits.total' field in the query response: " + responseMap); + return; + } + Integer total = ((Number) totalMap.get("value")).intValue(); // Extract the integer value + if (total == null) { - log.warn("Didn't find a 'hits.total' field " - + "in the query response: " + responseMap); + log.warn("Didn't find a 'hits.total.value' field in the query response: " + responseMap); return; } + totalHits = total; @SuppressWarnings("unchecked") - List> hits = (List>) uberHits - .get("hits"); + List> hits = (List>) uberHits.get("hits"); if (hits == null) { - log.warn("Didn't find a 'hits.hits' field " - + "in the query response: " + responseMap); + log.warn("Didn't find a 'hits.hits' field in the query response: " + responseMap); return; } parseDocuments(hits); } + private void parseDocuments(List> hits) { for (Map hit : hits) { SearchResultDocument doc = parseDocument(hit); @@ -141,8 +183,7 @@ private void parseDocuments(List> hits) { private SearchResultDocument parseDocument(Map hitMap) { @SuppressWarnings("unchecked") - Map> sourceMap = (Map>) hitMap - .get("_source"); + Map sourceMap = (Map) hitMap.get("_source"); if (sourceMap == null) { log.warn("Didn't find a '_source' field in the hit: " + hitMap); return null; @@ -154,14 +195,29 @@ private SearchResultDocument parseDocument(Map hitMap) { return null; } - return new BaseSearchResultDocument(id, sourceMap); + Map> parsedSourceMap = new HashMap<>(); + for (Map.Entry entry : sourceMap.entrySet()) { + Object value = entry.getValue(); + if (value instanceof Collection) { + parsedSourceMap.put(entry.getKey(), (Collection) value); + } else if (value instanceof Map) { + // This is done assuming the only "Map" field will be a _drsim field + parsedSourceMap.put(entry.getKey(), Collections.singletonList( + ((Map) value).get("gte") + " TO " + ((Map) value).get("lte")) + ); + } else { + parsedSourceMap.put(entry.getKey(), Collections.singletonList(value)); + } + } + + return new BaseSearchResultDocument(id, parsedSourceMap); } private Map> parseHighlight( - Map hitMap) { + Map hitMap) { @SuppressWarnings("unchecked") Map> highlightMap = (Map>) hitMap - .get("highlight"); + .get("highlight"); if (highlightMap == null) { log.debug("Didn't find a 'highlight' field in the hit: " + hitMap); return null; @@ -171,7 +227,7 @@ private Map> parseHighlight( List snippets = highlightMap.get("ALLTEXT"); if (snippets == null) { log.warn("Didn't find a 'highlight.ALLTEXT' field in the hit: " - + hitMap); + + hitMap); return null; } diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/elasticsearch/SearchType.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/elasticsearch/SearchType.java new file mode 100644 index 0000000000..d40cfc7d8f --- /dev/null +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/elasticsearch/SearchType.java @@ -0,0 +1,13 @@ +package edu.cornell.mannlib.vitro.webapp.searchengine.elasticsearch; + +public enum SearchType { + MATCH, + FUZZY, + PHRASE, + RANGE, + PREFIX, + WILDCARD, + EXISTS, + MATCH_ALL, + REGEXP +} diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/solr/SolrSearchEngine.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/solr/SolrSearchEngine.java index 3db3e2b749..cc26e599d2 100644 --- a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/solr/SolrSearchEngine.java +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/solr/SolrSearchEngine.java @@ -10,6 +10,8 @@ import javax.servlet.ServletContext; +import edu.cornell.mannlib.vitro.webapp.searchengine.base.SearchEngineUtil; +import edu.cornell.mannlib.vitro.webapp.servlet.setup.SearchEngineSmokeTest; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.impl.client.StandardHttpRequestRetryHandler; import org.apache.solr.client.solrj.SolrClient; @@ -45,10 +47,9 @@ public class SolrSearchEngine implements SearchEngine { @Override public void startup(Application application, ComponentStartupStatus css) { ServletContext ctx = application.getServletContext(); - String solrServerUrlString = ConfigurationProperties.getBean(ctx) - .getProperty("vitro.local.solr.url"); + String solrServerUrlString = SearchEngineUtil.getSearchEngineURLProperty(); if (solrServerUrlString == null) { - css.fatal("Could not find vitro.local.solr.url in " + css.fatal("Could not find vitro.local.searchengine.url in " + "runtime.properties. Vitro application needs the URL of " + "a solr server that it can use to index its data. It " + "should be something like http://localhost:${port}" diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/servlet/setup/ElasticSmokeTest.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/servlet/setup/ElasticSmokeTest.java index 7dabca6990..d6ac745d08 100644 --- a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/servlet/setup/ElasticSmokeTest.java +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/servlet/setup/ElasticSmokeTest.java @@ -4,7 +4,7 @@ import edu.cornell.mannlib.vitro.webapp.config.ConfigurationProperties; import edu.cornell.mannlib.vitro.webapp.startup.StartupStatus; -import edu.cornell.mannlib.vitro.webapp.utils.http.HttpClientFactory; +import edu.cornell.mannlib.vitro.webapp.utils.http.ESHttpBasicClientFactory; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.http.HttpResponse; @@ -15,6 +15,7 @@ import javax.servlet.ServletContextEvent; import javax.servlet.ServletContextListener; + import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; @@ -35,11 +36,11 @@ public ElasticSmokeTest(ServletContextListener listener) { public void doTest(ServletContextEvent sce) { final StartupStatus ss = StartupStatus.getBean(sce.getServletContext()); - String elasticUrlString = ConfigurationProperties.getBean(sce).getProperty("vitro.local.elastic.url", ""); + String elasticUrlString = ConfigurationProperties.getBean(sce).getProperty("vitro.local.searchengine.url", ""); if (elasticUrlString.isEmpty()) { ss.fatal(listener, "Can't connect to ElasticSearch engine. " - + "runtime.properties must contain a value for " - + "vitro.local.elastic.url"); + + "runtime.properties must contain a value for " + + "vitro.local.searchengine.url"); return; } @@ -49,9 +50,9 @@ public void doTest(ServletContextEvent sce) { elasticUrl = new URL(elasticUrlString); } catch (MalformedURLException e) { ss.fatal(listener, "Can't connect to ElasticSearch engine. " - + "The value for vitro.local.elastic.url " - + "in runtime.properties is not a valid URL: '" - + elasticUrlString + "'", e); + + "The value for vitro.local.searchengine.url " + + "in runtime.properties is not a valid URL: '" + + elasticUrlString + "'", e); } ss.info(listener, "Starting ElasticSearch test."); @@ -82,10 +83,11 @@ private void reportPingProblem(StartupStatus ss, ElasticProblemException e) { */ private static class ElasticPinger { private final URL elasticUrl; - private final HttpClient httpClient = HttpClientFactory.getHttpClient(); + private final HttpClient httpClient; public ElasticPinger(URL elasticUrl) { this.elasticUrl = elasticUrl; + this.httpClient = ESHttpBasicClientFactory.getHttpClient(elasticUrl.toString()); } public void ping() throws ElasticProblemException { diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/servlet/setup/SearchEngineSmokeTest.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/servlet/setup/SearchEngineSmokeTest.java index 8c6267a431..f3201694d4 100644 --- a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/servlet/setup/SearchEngineSmokeTest.java +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/servlet/setup/SearchEngineSmokeTest.java @@ -2,14 +2,26 @@ package edu.cornell.mannlib.vitro.webapp.servlet.setup; -import edu.cornell.mannlib.vitro.webapp.config.ConfigurationProperties; -import edu.cornell.mannlib.vitro.webapp.startup.StartupStatus; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import java.io.IOException; +import java.net.MalformedURLException; +import java.util.Objects; import javax.servlet.ServletContextEvent; import javax.servlet.ServletContextListener; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import edu.cornell.mannlib.vitro.webapp.searchengine.base.SearchEngineUtil; +import edu.cornell.mannlib.vitro.webapp.startup.StartupStatus; +import edu.cornell.mannlib.vitro.webapp.utils.http.ESHttpBasicClientFactory; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.http.HttpEntity; +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.util.EntityUtils; + /** * Start up the appropriate search engine smoke test based on the configured URL property. */ @@ -17,26 +29,88 @@ public class SearchEngineSmokeTest implements ServletContextListener { private static final Log log = LogFactory.getLog(SearchEngineSmokeTest.class); + private static ServiceType identifyService(String url) throws MalformedURLException { + String baseServiceUrl = getBaseServiceUrl(url); + + ServiceType serviceType = ServiceType.UNKNOWN; + HttpClient httpClient = ESHttpBasicClientFactory.getHttpClient(baseServiceUrl); + HttpGet request = new HttpGet(baseServiceUrl); + + try { + HttpResponse response = httpClient.execute(request); + HttpEntity entity = response.getEntity(); + if (entity != null) { + String result = EntityUtils.toString(entity); + + if (result.contains("Solr")) { + return ServiceType.SOLR; + } + + ObjectMapper mapper = new ObjectMapper(); + JsonNode rootNode = mapper.readTree(result); + + if (rootNode.has("version")) { + JsonNode versionNode = rootNode.get("version"); + if (versionNode.has("distribution") && + "opensearch".equals(versionNode.get("distribution").asText())) { + serviceType = ServiceType.OPENSEARCH; + } else if (versionNode.has("number")) { + serviceType = ServiceType.ELASTIC; + } + } + } + } catch (IOException e) { + System.err.println("Request failed: " + e.getMessage()); + } + + return serviceType; + } + + private static String getBaseServiceUrl(String url) { + if (url.endsWith("/")) { + url = url.substring(0, url.length() - 1); + } + + int lastSlashIndex = url.lastIndexOf('/'); + if (lastSlashIndex != -1) { + return url.substring(0, lastSlashIndex); + } + return url; + } + @Override public void contextInitialized(ServletContextEvent sce) { final StartupStatus ss = StartupStatus.getBean(sce.getServletContext()); - String solrUrlString = ConfigurationProperties.getBean(sce).getProperty("vitro.local.solr.url", ""); - String elasticUrlString = ConfigurationProperties.getBean(sce).getProperty("vitro.local.elastic.url", ""); + String searchEngineUrlString = SearchEngineUtil.getSearchEngineURLProperty(); - if (!solrUrlString.isEmpty() && !elasticUrlString.isEmpty()) { - ss.fatal(this, "More than one search engine is configured: " + solrUrlString + ", and " + elasticUrlString); - - } else if (solrUrlString.isEmpty() && elasticUrlString.isEmpty()) { + if (Objects.isNull(searchEngineUrlString) || searchEngineUrlString.isEmpty()) { ss.fatal(this, "No search engine is configured"); + } + + ServiceType service = ServiceType.UNKNOWN; + try { + service = identifyService(searchEngineUrlString); + } catch (MalformedURLException e) { + ss.fatal(this, "Search engine service URL is malformed."); + } - } else if (!solrUrlString.isEmpty()) { - log.debug("Initializing Solr: " + solrUrlString); - new SolrSmokeTest(this).doTest(sce); - } else { - log.debug("Initializing ElasticSearch: " + elasticUrlString); - new ElasticSmokeTest(this).doTest(sce); + switch (service) { + case ELASTIC: + log.debug("Initializing ElasticSearch: " + searchEngineUrlString); + new ElasticSmokeTest(this).doTest(sce); + break; + case OPENSEARCH: + log.debug("Initializing OpenSearch: " + searchEngineUrlString); + new ElasticSmokeTest(this).doTest(sce); + break; + case SOLR: + log.debug("Initializing Solr: " + searchEngineUrlString); + new SolrSmokeTest(this).doTest(sce); + break; + default: + ss.fatal(this, "Unknown search engine service is configured"); } } @@ -45,4 +119,11 @@ public void contextDestroyed(ServletContextEvent sce) { // nothing to tear down. } + private enum ServiceType { + ELASTIC, + OPENSEARCH, + SOLR, + UNKNOWN + } + } diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/servlet/setup/SolrSmokeTest.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/servlet/setup/SolrSmokeTest.java index 7b2536c1fb..0cca1f2d17 100644 --- a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/servlet/setup/SolrSmokeTest.java +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/servlet/setup/SolrSmokeTest.java @@ -7,10 +7,12 @@ import java.net.MalformedURLException; import java.net.URL; import java.net.UnknownHostException; +import java.util.Objects; import javax.servlet.ServletContextEvent; import javax.servlet.ServletContextListener; +import edu.cornell.mannlib.vitro.webapp.searchengine.base.SearchEngineUtil; import edu.cornell.mannlib.vitro.webapp.utils.http.HttpClientFactory; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -20,7 +22,6 @@ import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpGet; -import edu.cornell.mannlib.vitro.webapp.config.ConfigurationProperties; import edu.cornell.mannlib.vitro.webapp.startup.StartupStatus; import edu.cornell.mannlib.vitro.webapp.utils.threads.VitroBackgroundThread; import org.apache.http.util.EntityUtils; @@ -52,12 +53,11 @@ public SolrSmokeTest(ServletContextListener listener) { public void doTest(ServletContextEvent sce) { final StartupStatus ss = StartupStatus.getBean(sce.getServletContext()); - String solrUrlString = ConfigurationProperties.getBean(sce) - .getProperty("vitro.local.solr.url", ""); - if (solrUrlString.isEmpty()) { + String solrUrlString = SearchEngineUtil.getSearchEngineURLProperty(); + if (Objects.isNull(solrUrlString) || solrUrlString.isEmpty()) { ss.fatal(listener, "Can't connect to Solr search engine. " + "runtime.properties must contain a value for " - + "vitro.local.solr.url"); + + "vitro.local.searchengine.url (vitro.local.solr.url)"); return; } @@ -67,7 +67,7 @@ public void doTest(ServletContextEvent sce) { solrUrl = new URL(solrUrlString); } catch (MalformedURLException e) { ss.fatal(listener, "Can't connect to Solr search engine. " - + "The value for vitro.local.solr.url " + + "The value for vitro.local.searchengine.url (vitro.local.solr.url) " + "in runtime.properties is not a valid URL: '" + solrUrlString + "'", e); } @@ -151,14 +151,14 @@ private void reportPingProblem(SolrProblemException e) { private void warnSocketTimeout() { ss.warning(listener, "Can't connect to the Solr search engine. " + "The socket connection has repeatedly timed out. " - + "Check the value of vitro.local.solr.url in " + + "Check the value of vitro.local.searchengine.url (vitro.local.solr.url) in " + "runtime.properties. Is Solr responding at that URL?"); } private void warnBadHttpStatus(int status) { ss.warning(listener, "Can't connect to the Solr search engine. " + "The Solr server returned a status code of " + status - + ". Check the value of vitro.local.solr.url in " + + ". Check the value of vitro.local.searchengine.url (vitro.local.solr.url) in " + "runtime.properties."); } @@ -170,7 +170,7 @@ private void warnProtocolViolation(HttpException e) { private void warnUnknownHost(UnknownHostException e) { ss.warning(listener, "Can't connect to the Solr search engine. '" + e.getMessage() + "' is an unknown host." - + "Check the value of vitro.local.solr.url in " + + "Check the value of vitro.local.searchengine.url (vitro.local.solr.url) in " + "runtime.properties.", e); } @@ -178,7 +178,7 @@ private void warnConnectionRefused(ConnectException e) { ss.warning(listener, "Can't connect to the Solr search engine. " + "The host refused the connection. " + "Is it possible that the port number is incorrect? " - + "Check the value of vitro.local.solr.url in " + + "Check the value of vitro.local.searchengine.url (vitro.local.solr.url) in " + "runtime.properties.", e); } diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/utils/http/ESHttpBasicClientFactory.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/utils/http/ESHttpBasicClientFactory.java new file mode 100644 index 0000000000..4405981b55 --- /dev/null +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/utils/http/ESHttpBasicClientFactory.java @@ -0,0 +1,133 @@ +package edu.cornell.mannlib.vitro.webapp.utils.http; + +import java.util.concurrent.TimeUnit; + +import javax.net.ssl.SSLContext; + +import edu.cornell.mannlib.vitro.webapp.config.ConfigurationProperties; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.config.RegistryBuilder; +import org.apache.http.conn.socket.ConnectionSocketFactory; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.apache.http.ssl.SSLContextBuilder; + +public class ESHttpBasicClientFactory { + + + private static final Log log = LogFactory.getLog(ESHttpBasicClientFactory.class); + + private static volatile CloseableHttpClient httpClient; + + private static volatile CloseableHttpClient httpsClient; + + private static volatile PoolingHttpClientConnectionManager connectionManager; + + + public static CloseableHttpClient getHttpClient(String baseUrl) { + boolean isHttps = baseUrl.startsWith("https"); + + if (isHttps) { + return getHttpsClient(); + } else { + return getHttpClient(); + } + } + + public static CloseableHttpClient getHttpClient() { + if (httpClient == null) { + synchronized (ESHttpBasicClientFactory.class) { + if (httpClient == null) { + httpClient = createHttpClient(false); + } + } + } + return httpClient; + } + + public static CloseableHttpClient getHttpsClient() { + if (httpsClient == null) { + synchronized (ESHttpBasicClientFactory.class) { + if (httpsClient == null) { + httpsClient = createHttpClient(true); + } + } + } + return httpsClient; + } + + private static synchronized PoolingHttpClientConnectionManager getConnectionManager(boolean isHttps) { + if (connectionManager == null) { + if (isHttps) { + try { + SSLContext sslContext = SSLContextBuilder.create() + .loadTrustMaterial(null, (chain, authType) -> true) // Trust all certificates + .build(); + + connectionManager = new PoolingHttpClientConnectionManager( + RegistryBuilder.create() + .register("https", new SSLConnectionSocketFactory(sslContext, + (hostname, session) -> true)) // Allow all hostnames + .build() + ); + } catch (Exception e) { + throw new RuntimeException("Failed to create SSL connection manager", e); + } + } else { + connectionManager = new PoolingHttpClientConnectionManager(); + } + + connectionManager.setDefaultMaxPerRoute(50); + connectionManager.setMaxTotal(300); + connectionManager.setValidateAfterInactivity(30000); + } + return connectionManager; + } + + private static CloseableHttpClient createHttpClient(boolean isHttps) { + String elasticUsername = ConfigurationProperties.getInstance() + .getProperty("vitro.local.searchengine.username", ""); + String elasticPassword = ConfigurationProperties.getInstance() + .getProperty("vitro.local.searchengine.password", ""); + + boolean hasCredentials = !elasticUsername.isEmpty() && !elasticPassword.isEmpty(); + + if (isHttps && !hasCredentials) { + log.warn("Using HTTPS without authentication. This is not recommended for production."); + } + + HttpClientBuilder builder = HttpClients.custom() + .setConnectionManager(getConnectionManager(isHttps)) + .setConnectionManagerShared(true); + + if (hasCredentials) { + CredentialsProvider credsProvider = new BasicCredentialsProvider(); + credsProvider.setCredentials( + new AuthScope(AuthScope.ANY_HOST, AuthScope.ANY_PORT), + new UsernamePasswordCredentials(elasticUsername, elasticPassword) + ); + builder.setDefaultCredentialsProvider(credsProvider); + } + + RequestConfig requestConfig = RequestConfig.custom() + .setConnectTimeout(30000) // 30 seconds + .setSocketTimeout(60000) // 60 seconds + .setConnectionRequestTimeout(30000) // 30 seconds + .setCircularRedirectsAllowed(true) + .build(); + + return builder + .setDefaultRequestConfig(requestConfig) + .setConnectionTimeToLive(60, TimeUnit.SECONDS) // TTL for persistent connections + .build(); + } +} diff --git a/api/src/test/java/edu/cornell/mannlib/vitro/webapp/searchengine/searchquery/CustomQueryBuilderTest.java b/api/src/test/java/edu/cornell/mannlib/vitro/webapp/searchengine/searchquery/CustomQueryBuilderTest.java new file mode 100644 index 0000000000..b0ad42bc6a --- /dev/null +++ b/api/src/test/java/edu/cornell/mannlib/vitro/webapp/searchengine/searchquery/CustomQueryBuilderTest.java @@ -0,0 +1,202 @@ +package edu.cornell.mannlib.vitro.webapp.searchengine.searchquery; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import co.elastic.clients.elasticsearch._types.query_dsl.Query; +import edu.cornell.mannlib.vitro.webapp.searchengine.elasticsearch.CustomQueryBuilder; +import edu.cornell.mannlib.vitro.webapp.searchengine.elasticsearch.SearchType; +import org.junit.Test; + +public class CustomQueryBuilderTest { + + @Test + public void testBuildQueryMatchType() { + Query query = CustomQueryBuilder.buildQuery(SearchType.MATCH, "title", "test value"); + + assertNotNull("Query should not be null", query); + assertTrue("Should be MatchQuery", query.isMatch()); + assertEquals("Field should match", "title", getQueryField(query)); + assertEquals("Value should match", "test value", getQueryValue(query)); + } + + @Test + public void testBuildQueryFuzzyType() { + Query query = CustomQueryBuilder.buildQuery(SearchType.FUZZY, "name", "test~"); + + assertNotNull("Query should not be null", query); + assertTrue("Should be FuzzyQuery", query.isFuzzy()); + assertEquals("Field should match", "name", query.fuzzy().field()); + assertEquals("Value should have tilde removed", "test", getQueryValue(query)); + assertEquals("Fuzziness should be 2", "2", query.fuzzy().fuzziness()); + } + + @Test + public void testBuildQueryPrefixType() { + Query query = CustomQueryBuilder.buildQuery(SearchType.PREFIX, "category", "pref"); + + assertNotNull("Query should not be null", query); + assertTrue("Should be PrefixQuery", query.isPrefix()); + assertEquals("Field should match", "category", getQueryField(query)); + assertEquals("Value should match", "pref", getQueryValue(query)); + } + + @Test + public void testBuildQueryRangeType() { + Query query = CustomQueryBuilder.buildQuery(SearchType.RANGE, "price", "[100 TO 200]"); + + assertNotNull("Query should not be null", query); + assertTrue("Should be RangeQuery", query.isRange()); + assertEquals("Field should match", "price", query.range().field()); + assertEquals("From value should be cleaned", "100", query.range().from()); + assertEquals("To value should be cleaned", "200", query.range().to()); + } + + @Test + public void testBuildQueryExistsType() { + Query query = CustomQueryBuilder.buildQuery(SearchType.EXISTS, "description", "anyValue"); + + assertNotNull("Query should not be null", query); + assertTrue("Should be ExistsQuery", query.isExists()); + assertEquals("Field should match", "description", getQueryField(query)); + } + + @Test + public void testBuildQueryMatchAllType() { + Query query = CustomQueryBuilder.buildQuery(SearchType.MATCH_ALL, "anyField", "anyValue"); + + assertNotNull("Query should not be null", query); + assertTrue("Should be MatchAllQuery", query.isMatchAll()); + } + + @Test + public void testBuildQueryRegexpTypeWithSolrFormat() { + Query query = CustomQueryBuilder.buildQuery(SearchType.REGEXP, "content", "/test.*pattern/"); + + assertNotNull("Query should not be null", query); + assertTrue("Should be RegexpQuery", query.isRegexp()); + assertEquals("Field should match", "content", getQueryField(query)); + assertEquals("Value should be cleaned", "test.*pattern", getQueryValue(query)); + } + + @Test + public void testBuildQueryRegexpTypeWithoutSolrFormat() { + Query query = CustomQueryBuilder.buildQuery(SearchType.REGEXP, "content", "test.*pattern"); + + assertNotNull("Query should not be null", query); + assertTrue("Should be RegexpQuery", query.isRegexp()); + assertEquals("Value should remain unchanged", "test.*pattern", getQueryValue(query)); + } + + @Test + public void testBuildQueryWildcardType() { + Query query = CustomQueryBuilder.buildQuery(SearchType.WILDCARD, "title", "test.*"); + + assertNotNull("Query should not be null", query); + assertTrue("Should be WildcardQuery", query.isWildcard()); + assertEquals("Field should match", "title", getQueryField(query)); + assertEquals("Value should be converted", "test*", getQueryValue(query)); + } + + @Test + public void testBuildQueryWildcardTypeWithStarField() { + Query query = CustomQueryBuilder.buildQuery(SearchType.WILDCARD, "*", "anyValue"); + + assertNotNull("Query should not be null", query); + assertTrue("Should be MatchAllQuery when field is *", query.isMatchAll()); + } + + @Test + public void testBuildQueryDefaultType() { + Query query = CustomQueryBuilder.buildQuery(SearchType.PHRASE, "content", "\"exact phrase\""); + + assertNotNull("Query should not be null", query); + assertTrue("Should be MatchPhraseQuery", query.isMatchPhrase()); + assertEquals("Field should match", "content", getQueryField(query)); + assertEquals("Value should be cleaned", "exact phrase", getQueryValue(query)); + } + + @Test + public void testBuildQueryDefaultTypeWithShortPhrase() { + Query query = CustomQueryBuilder.buildQuery(SearchType.PHRASE, "content", "\"a\""); + + assertNotNull("Query should not be null", query); + assertEquals("Short phrase should remain", "a", getQueryValue(query)); + } + + @Test(expected = IllegalArgumentException.class) + public void testBuildQueryWithNullField() { + CustomQueryBuilder.buildQuery(SearchType.MATCH, null, "value"); + } + + @Test(expected = IllegalArgumentException.class) + public void testBuildQueryWithEmptyField() { + CustomQueryBuilder.buildQuery(SearchType.MATCH, "", "value"); + } + + @Test(expected = IllegalArgumentException.class) + public void testBuildQueryWithNullValue() { + CustomQueryBuilder.buildQuery(SearchType.MATCH, "field", null); + } + + @Test + public void testBuildQueryWithEmptyValue() { + Query query = CustomQueryBuilder.buildQuery(SearchType.MATCH, "field", ""); + + assertNotNull("Query should not be null", query); + assertTrue("Should be MatchQuery", query.isMatch()); + assertEquals("Empty value should be preserved", "", getQueryValue(query)); + } + + @Test + public void testBuildQueryRangeTypeWithDifferentBrackets() { + Query query = CustomQueryBuilder.buildQuery(SearchType.RANGE, "date", "(2020 TO 2023]"); + + assertNotNull("Query should not be null", query); + assertEquals("From value should be cleaned", "2020", query.range().from()); + assertEquals("To value should be cleaned", "2023", query.range().to()); + } + + // ---------------------------------------------------------------------- + // Helper methods + // ---------------------------------------------------------------------- + + private String getQueryValue(Query query) { + if (query.isMatch()) { + return query.match().query().stringValue(); + } else if (query.isFuzzy()) { + return query.fuzzy().value().stringValue(); + } else if (query.isPrefix()) { + return query.prefix().value(); + } else if (query.isRegexp()) { + return query.regexp().value(); + } else if (query.isWildcard()) { + return query.wildcard().value(); + } else if (query.isMatchPhrase()) { + return query.matchPhrase().query(); + } + return null; + } + + private String getQueryField(Query query) { + if (query.isMatch()) { + return query.match().field(); + } else if (query.isFuzzy()) { + return query.fuzzy().field(); + } else if (query.isPrefix()) { + return query.prefix().field(); + } else if (query.isRegexp()) { + return query.regexp().field(); + } else if (query.isWildcard()) { + return query.wildcard().field(); + } else if (query.isMatchPhrase()) { + return query.matchPhrase().field(); + } else if (query.isRange()) { + return query.range().field(); + } else if (query.isExists()) { + return query.exists().field(); + } + return null; + } +} diff --git a/api/src/test/java/edu/cornell/mannlib/vitro/webapp/searchengine/searchquery/ExpressionTransformerTest.java b/api/src/test/java/edu/cornell/mannlib/vitro/webapp/searchengine/searchquery/ExpressionTransformerTest.java new file mode 100644 index 0000000000..e5f2f951ac --- /dev/null +++ b/api/src/test/java/edu/cornell/mannlib/vitro/webapp/searchengine/searchquery/ExpressionTransformerTest.java @@ -0,0 +1,94 @@ +package edu.cornell.mannlib.vitro.webapp.searchengine.searchquery; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import java.util.List; + +import edu.cornell.mannlib.vitro.webapp.searchengine.elasticsearch.ExpressionTransformer; +import org.junit.Test; + +public class ExpressionTransformerTest { + + @Test + public void testRemoveWhitespacesFromRangeExpression_RemovesSpaces() { + String input = "[ 2020-01-01 TO 2020-12-31 ]"; + String expected = "[2020-01-01TO2020-12-31]"; + + String actual = ExpressionTransformer.removeWhitespacesFromRangeExpression(input); + + assertEquals(expected, actual); + } + + @Test + public void testRemoveWhitespacesFromRangeExpression_NoRange() { + String input = "title:hello"; + assertEquals(input, ExpressionTransformer.removeWhitespacesFromRangeExpression(input)); + } + + @Test + public void testRemoveInvalidParentheses_RemovesUnmatchedClosing() { + List tokens = java.util.Arrays.asList("(", "foo", ")", ")"); + List cleaned = ExpressionTransformer.removeInvalidParentheses(tokens); + + assertEquals(java.util.Arrays.asList("(", "foo", ")"), cleaned); + } + + @Test + public void testRemoveInvalidParentheses_PreservesInsideQuotes() { + List tokens = java.util.Collections.singletonList("\"(hello world)\""); + List cleaned = ExpressionTransformer.removeInvalidParentheses(tokens); + + assertEquals(java.util.Collections.singletonList("\"(hello world)\""), cleaned); + } + + @Test + public void testIsTokenAPredefinedFieldQuery_TrueForField() { + assertTrue(ExpressionTransformer.isTokenAPredefinedFieldQuery("title:hello")); + } + + @Test + public void testIsTokenAPredefinedFieldQuery_FalseNoColon() { + assertFalse(ExpressionTransformer.isTokenAPredefinedFieldQuery("hello")); + } + + @Test + public void testIsTokenAPredefinedFieldQuery_FalseWhenStartsOrEndsWithColon() { + assertFalse(ExpressionTransformer.isTokenAPredefinedFieldQuery(":bad")); + assertFalse(ExpressionTransformer.isTokenAPredefinedFieldQuery("bad:")); + } + + @Test + public void testFillInMissingOperators_WrapsPlainTerms() { + String query = "alpha beta"; + String result = ExpressionTransformer.fillInMissingOperators(query); + + assertTrue(result.contains("ALLTEXT:alpha")); + assertTrue(result.contains("nameLowercaseSingleValued:alpha")); + } + + @Test + public void testFillInMissingOperators_LeavesFieldQuery() { + String query = "title:java"; + String result = ExpressionTransformer.fillInMissingOperators(query); + + assertTrue(result.contains("title:java")); + assertFalse(result.contains("ALLTEXT")); + } + + @Test + public void testParseAdvancedQuery_ReturnsNonNullForSimpleExpression() { + List expression = java.util.Collections.singletonList("ALLTEXT:java"); + + assertNotNull(new ExpressionTransformer().parseAdvancedQuery(expression)); + } + + @Test + public void testParseAdvancedQuery_WithLogicalOperators() { + List expression = java.util.Arrays.asList("field1:value1", "AND", "field2:value2"); + + assertNotNull(new ExpressionTransformer().parseAdvancedQuery(expression)); + } +} diff --git a/api/src/test/java/edu/cornell/mannlib/vitro/webapp/searchengine/util/SearchEngineUtilTest.java b/api/src/test/java/edu/cornell/mannlib/vitro/webapp/searchengine/util/SearchEngineUtilTest.java new file mode 100644 index 0000000000..710501454d --- /dev/null +++ b/api/src/test/java/edu/cornell/mannlib/vitro/webapp/searchengine/util/SearchEngineUtilTest.java @@ -0,0 +1,67 @@ +package edu.cornell.mannlib.vitro.webapp.searchengine.util; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +import edu.cornell.mannlib.vitro.webapp.config.ConfigurationProperties; +import edu.cornell.mannlib.vitro.webapp.searchengine.base.SearchEngineUtil; +import org.junit.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +public class SearchEngineUtilTest { + + @Test + public void returnsConfiguredSearchEngineUrl() { + try (MockedStatic mocked = Mockito.mockStatic(ConfigurationProperties.class)) { + ConfigurationProperties config = Mockito.mock(ConfigurationProperties.class); + + mocked.when(ConfigurationProperties::getInstance).thenReturn(config); + Mockito.when(config.getProperty("vitro.local.searchengine.url", "")).thenReturn("http://search:8983/solr"); + + String result = SearchEngineUtil.getSearchEngineURLProperty(); + + assertEquals("http://search:8983/solr", result); + } + } + + @Test + public void fallsBackToLegacySolrUrlAndWarns() { + try (MockedStatic mocked = Mockito.mockStatic(ConfigurationProperties.class)) { + ConfigurationProperties config = Mockito.mock(ConfigurationProperties.class); + + mocked.when(ConfigurationProperties::getInstance).thenReturn(config); + Mockito.when(config.getProperty("vitro.local.searchengine.url", "")).thenReturn(""); + Mockito.when(config.getProperty("vitro.local.solr.url", "")).thenReturn("http://legacy:8983/solr"); + + String result = SearchEngineUtil.getSearchEngineURLProperty(); + + assertEquals("http://legacy:8983/solr", result); + } + } + + @Test + public void returnsNullWhenConfigIsNull() { + try (MockedStatic mocked = Mockito.mockStatic(ConfigurationProperties.class)) { + mocked.when(ConfigurationProperties::getInstance).thenReturn(null); + + assertNull(SearchEngineUtil.getSearchEngineURLProperty()); + } + } + + @Test + public void returnsEmptyWhenNoConfigValues() { + try (MockedStatic mocked = Mockito.mockStatic(ConfigurationProperties.class)) { + ConfigurationProperties config = Mockito.mock(ConfigurationProperties.class); + + mocked.when(ConfigurationProperties::getInstance).thenReturn(config); + Mockito.when(config.getProperty("vitro.local.searchengine.url", "")).thenReturn(""); + Mockito.when(config.getProperty("vitro.local.solr.url", "")).thenReturn(""); + + String result = SearchEngineUtil.getSearchEngineURLProperty(); + + assertEquals("", result); + } + } +} + diff --git a/home/src/main/resources/config/example.applicationSetup.n3 b/home/src/main/resources/config/example.applicationSetup.n3 index d5beb3c9ca..817fe2c398 100644 --- a/home/src/main/resources/config/example.applicationSetup.n3 +++ b/home/src/main/resources/config/example.applicationSetup.n3 @@ -64,7 +64,7 @@ # ---------------------------- # # Search engine module: -# The Solr-based implementation is the only standard option, but it can be +# The Solr-based implementation is the standard option and it can be # wrapped in an "instrumented" wrapper, which provides additional logging # and more rigorous life-cycle checking. # @@ -78,6 +78,17 @@ a vitroWebapp:searchengine.solr.SolrSearchEngine , vitroWebapp:modules.searchEngine.SearchEngine . +# Alternatively, you can setup Elasticsearch engine in the same manner: + +#:instrumentedSearchEngineWrapper +# a vitroWebapp:searchengine.InstrumentedSearchEngineWrapper , +# vitroWebapp:modules.searchEngine.SearchEngine ; +# :wraps :elasticSearchEngine . + +#:elasticSearchEngine +# a vitroWebapp:searchengine.elasticsearch.ElasticSearchEngine , +# vitroWebapp:modules.searchEngine.SearchEngine . + # ---------------------------- # # Search indexer module: diff --git a/home/src/main/resources/config/example.runtime.properties b/home/src/main/resources/config/example.runtime.properties index 074941b2bb..1de4de8329 100644 --- a/home/src/main/resources/config/example.runtime.properties +++ b/home/src/main/resources/config/example.runtime.properties @@ -22,19 +22,29 @@ # Vitro.defaultNamespace = http://vivo.mydomain.edu/individual/ + # To exclude the Tomcat application context name from generated links uncomment following line. # Default value is false. # context.path.exclude = true -# URL of Solr context used in local Vitro search. This will usually consist of: +# WARNING: Deprecated, switch to using vitro.local.searchengine.url as explained below +# URL of Solr context used in local VIVO search. This will usually consist of: # scheme + server_name + port + "solr" + solr_core_name # In a standard Solr installation, the Solr service will be available on port # 8983. The path will be /solr followed by the name used when adding a core -# for Vitro. +# for VIVO. # Example: -# vitro.local.solr.url = http://localhost:8983/solr/vitrocore +# vitro.local.solr.url = http://localhost:8983/solr/vivocore # -vitro.local.solr.url = http://localhost:8983/solr/vitrocore +vitro.local.solr.url = http://localhost:8983/solr/vivocore + +# URL of generic searchengine context (Supports Solr, Elasticsearch and Opensearch) +# vitro.local.searchengine.url = http://localhost:8983/solr/vivocore + +# If basic authentication is supported for connection to you search engine instance, +# the credentials should be provided below +#vitro.local.searchengine.username = +#vitro.local.searchengine.password = # # Email parameters Vitro uses to send email. If these are left empty,