Elasticsearch 7.* and 8.* integration. OpenSearch integration.

api/pom.xml

@@ -102,6 +102,11 @@
             <artifactId>guava</artifactId>
             <version>32.0.0-jre</version>
         </dependency>
+        <dependency>
+            <groupId>co.elastic.clients</groupId>
+            <artifactId>elasticsearch-java</artifactId>
+            <version>8.9.0</version>
+        </dependency>
         <dependency>
             <groupId>org.apache.httpcomponents</groupId>
             <artifactId>fluent-hc</artifactId>

api/src/main/java/edu/cornell/mannlib/vitro/webapp/modules/searchEngine/SearchQuery.java

@@ -161,4 +161,8 @@ public enum Order {
      * @return defines whether the text of a facet field should be compared case insensitively.
      */
     boolean isFacetTextCompareCaseInsensitive();
+
+	void setSimpleQuery(boolean value);
+
+	boolean isSimpleQuery();
 }

api/src/main/java/edu/cornell/mannlib/vitro/webapp/search/controller/PagedSearchController.java

@@ -172,7 +172,7 @@ public static ResponseValues process(VitroRequest vreq, Map<String, List<String>
             int hitsPerPage = getHitsPerPage(vreq);
             int documentsToReturn = hitsPerPage;
             if (!wasHtmlRequested) {
-                documentsToReturn = getDocumentsNumber(vreq);    
+                documentsToReturn = getDocumentsNumber(vreq);
             }
             String queryText = getQueryText(vreq);
             log.debug("Query text is \"" + queryText + "\"");
@@ -203,6 +203,8 @@ public static ResponseValues process(VitroRequest vreq, Map<String, List<String>
             SearchResponse response = null;
 
             try {
+                query.setSimpleQuery(!query.getQuery().trim().equals("*:*"));
+
                 response = search.query(query);
             } catch (Exception ex) {
                 String msg = makeBadSearchMessage(queryText, ex.getMessage(), vreq);
@@ -259,7 +261,7 @@ public static ResponseValues process(VitroRequest vreq, Map<String, List<String>
             /* Compile the data for the templates */
 
             Map<String, Object> body = new HashMap<String, Object>();
-          
+
             /* Add ClassGroup and type refinement links to body */
             if (wasHtmlRequested) {
                 if (log.isDebugEnabled()) {
@@ -415,7 +417,7 @@ private static int getHitsPerPage(VitroRequest vreq) {
         log.debug("hitsPerPage is " + hitsPerPage);
         return hitsPerPage;
     }
-    
+
     private static int getDocumentsNumber(VitroRequest vreq) {
         int documentsNumber = DEFAULT_DOCUMENTS_NUMBER;
         try {

api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/base/BaseSearchQuery.java

@@ -29,6 +29,7 @@ public class BaseSearchQuery implements SearchQuery {
 	private int facetMinCount = -1;
     private boolean facetTextToCompareIgnoreCase;
     private String facetContainsText;
+	private boolean simpleQuery = false;
 
 	@Override
 	public SearchQuery setQuery(String query) {
@@ -171,4 +172,13 @@ public boolean isFacetTextCompareCaseInsensitive() {
         return facetTextToCompareIgnoreCase;
     }
 
+	@Override
+	public void setSimpleQuery(boolean value) {
+		this.simpleQuery = value;
+	}
+
+	@Override
+	public boolean isSimpleQuery() {
+		return this.simpleQuery;
+	}
 }

api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/base/SearchEngineUtil.java

@@ -0,0 +1,39 @@
+package edu.cornell.mannlib.vitro.webapp.searchengine.base;
+
+import java.util.Objects;
+
+import javax.annotation.Nullable;
+
+import edu.cornell.mannlib.vitro.webapp.config.ConfigurationProperties;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+public class SearchEngineUtil {
+
+    private static final Log log = LogFactory.getLog(SearchEngineUtil.class);
+
+    @Nullable
+    public static String getSearchEngineURLProperty() {
+        ConfigurationProperties config = ConfigurationProperties.getInstance();
+        if (Objects.isNull(config)) {
+            return null;
+        }
+
+        if (config.getProperty("vitro.local.searchengine.url", "").isEmpty()) {
+            return tryFetchLegacySolrConfiguration(config);
+        }
+
+        return config.getProperty("vitro.local.searchengine.url", "");
+    }
+
+    private static String tryFetchLegacySolrConfiguration(ConfigurationProperties config) {
+        String legacyConfigValue = config.getProperty("vitro.local.solr.url", "");
+        if (!legacyConfigValue.isEmpty()) {
+            log.warn(
+                "vitro.local.solr.url is deprecated, switch to using" +
+                    " vitro.local.searchengine.url as soon as possible.");
+        }
+
+        return legacyConfigValue;
+    }
+}

api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/elasticsearch/CustomQueryBuilder.java

@@ -0,0 +1,95 @@
+package edu.cornell.mannlib.vitro.webapp.searchengine.elasticsearch;
+
+import co.elastic.clients.elasticsearch._types.query_dsl.ExistsQuery;
+import co.elastic.clients.elasticsearch._types.query_dsl.FuzzyQuery;
+import co.elastic.clients.elasticsearch._types.query_dsl.MatchAllQuery;
+import co.elastic.clients.elasticsearch._types.query_dsl.MatchPhraseQuery;
+import co.elastic.clients.elasticsearch._types.query_dsl.MatchQuery;
+import co.elastic.clients.elasticsearch._types.query_dsl.PrefixQuery;
+import co.elastic.clients.elasticsearch._types.query_dsl.Query;
+import co.elastic.clients.elasticsearch._types.query_dsl.RangeQuery;
+import co.elastic.clients.elasticsearch._types.query_dsl.RegexpQuery;
+import co.elastic.clients.elasticsearch._types.query_dsl.WildcardQuery;
+
+public class CustomQueryBuilder {
+
+    private static final String MAX_FUZZY_EDITS = "2";
+
+    private CustomQueryBuilder() {
+    }
+
+
+    public static Query buildQuery(SearchType queryType, String field, String value) {
+        validateInput(field, value);
+
+        switch (queryType) {
+            case MATCH:
+                return MatchQuery.of(m -> m
+                    .field(field)
+                    .query(value)
+                )._toQuery();
+            case FUZZY:
+                return FuzzyQuery.of(m -> m
+                    .field(field)
+                    .value(value.replace("~", ""))
+                    .fuzziness(MAX_FUZZY_EDITS)
+                )._toQuery();
+            case PREFIX:
+                return PrefixQuery.of(m -> m
+                    .field(field)
+                    .value(value)
+                )._toQuery();
+            case RANGE:
+                String[] values = value.split("TO");
+                return RangeQuery.of(m -> m
+                    .field(field)
+                    .from(values[0].replace("[", "").replace("(", "").trim())
+                    .to(values[1].replace("]", "").replace(")", "").trim())
+                )._toQuery();
+            case EXISTS:
+                return ExistsQuery.of(m -> m
+                    .field(field)
+                )._toQuery();
+            case MATCH_ALL:
+                return MatchAllQuery.of(m -> m)._toQuery();
+            case REGEXP:
+                String regexpValue;
+
+                boolean isSolrRegexpSpecification = value.startsWith("/") && value.endsWith("/") && value.length() > 1;
+                if (isSolrRegexpSpecification) {
+                    regexpValue = value.substring(1, value.length() - 1);
+                } else {
+                    regexpValue = value;
+                }
+
+                return RegexpQuery.of(m -> m
+                    .field(field)
+                    .value(regexpValue)
+                )._toQuery();
+            case WILDCARD:
+                if (field.trim().equals("*")) {
+                    return MatchAllQuery.of(m -> m)._toQuery();
+                }
+
+                return WildcardQuery.of(m -> m
+                    .field(field)
+                    .value(value.replace(".*", "*"))
+                )._toQuery();
+            default:
+                return MatchPhraseQuery.of(m -> m
+                        .field(field)
+                        .query(value.length() > 1 ? value.substring(1, value.length() - 1) : value)
+                    // Remove leading and trailing '"' character
+                )._toQuery();
+        }
+    }
+
+    private static void validateInput(String field, String value) {
+        if (field == null || field.isEmpty()) {
+            throw new IllegalArgumentException("Field not specified");
+        }
+        if (value == null) {
+            throw new IllegalArgumentException("Value not specified");
+        }
+    }
+}

api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/elasticsearch/ESAdder.java

@@ -8,18 +8,22 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.http.client.fluent.Request;
-import org.apache.http.client.fluent.Response;
-import org.apache.http.entity.ContentType;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
-
 import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngineException;
 import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchInputDocument;
 import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchInputField;
+import edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames;
+import edu.cornell.mannlib.vitro.webapp.utils.http.ESHttpBasicClientFactory;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpPut;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.util.EntityUtils;
 
 /**
  * The nuts and bolts of adding a document to the Elasticsearch index
@@ -44,7 +48,16 @@ private void addDocument(SearchInputDocument doc)
             throws SearchEngineException {
         try {
             Map<String, List<Object>> map = convertDocToMap(doc);
+
+            if (map.containsKey(VitroSearchTermNames.NAME_RAW)) {
+                map.putIfAbsent(VitroSearchTermNames.AC_NAME_STEMMED, map.get(VitroSearchTermNames.NAME_RAW));
+                map.putIfAbsent(VitroSearchTermNames.AC_NAME_UNTOKENIZED, map.get(VitroSearchTermNames.NAME_RAW));
+            }
+
             String json = new ObjectMapper().writeValueAsString(map);
+            if (json.contains("_drsim")) {
+                json = reformatDRSIMFields(json);
+            }
             log.debug("Adding document for '" + doc.getField("DocId") + "': "
                     + json);
 
@@ -75,15 +88,46 @@ private Map<String, List<Object>> convertDocToMap(SearchInputDocument doc) {
         return map;
     }
 
+    private String reformatDRSIMFields(String json) {
+        String patternString = "\\[(\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}Z) TO (\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}Z)]";
+        Pattern pattern = Pattern.compile(patternString);
+        Matcher matcher = pattern.matcher(json);
+
+        StringBuffer result = new StringBuffer();
+
+        while (matcher.find()) {
+            String dateStart = matcher.group(1);
+            String dateEnd = matcher.group(2);
+
+            String replacement = String.format("{\"gte\": \"%s\", \"lte\": \"%s\"}", dateStart, dateEnd)
+                .replace("{", "\\{")
+                .replace("}", "\\}");
+
+            matcher.appendReplacement(result, replacement);
+        }
+
+        matcher.appendTail(result);
+        return result.toString().replace("[\"{", "{").replace("}\"]", "}");
+    }
+
     private void putToElastic(String json, String docId)
             throws SearchEngineException {
         try {
             String url = baseUrl + "/_doc/"
                     + URLEncoder.encode(docId, "UTF8");
-            Response response = Request.Put(url)
-                    .bodyString(json, ContentType.APPLICATION_JSON).execute();
-            log.debug("Response from Elasticsearch: "
-                    + response.returnContent().asString());
+            HttpClient httpClient = ESHttpBasicClientFactory.getHttpClient(baseUrl);
+
+            HttpPut request = new HttpPut(url);
+            request.addHeader("Content-Type", "application/json");
+            request.setEntity(new StringEntity(json, "UTF-8"));
+            HttpResponse response = httpClient.execute(request);
+            if (response.getStatusLine().getStatusCode() >= 400) {
+                log.warn("Response from Elasticsearch: "
+                    + EntityUtils.toString(response.getEntity()));
+            } else {
+                log.debug("Response from Elasticsearch: "
+                    + EntityUtils.toString(response.getEntity()));
+            }
         } catch (Exception e) {
             throw new SearchEngineException("Failed to put to Elasticsearch",
                     e);

api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/elasticsearch/ESCounter.java

@@ -5,12 +5,13 @@
 import java.util.HashMap;
 import java.util.Map;
 
-import org.apache.http.client.fluent.Request;
-import org.apache.http.client.fluent.Response;
-
 import com.fasterxml.jackson.databind.ObjectMapper;
-
 import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngineException;
+import edu.cornell.mannlib.vitro.webapp.utils.http.ESHttpBasicClientFactory;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.util.EntityUtils;
 
 /**
  * The nuts and bolts of getting the number of documents in the Elasticsearch
@@ -25,9 +26,10 @@ public ESCounter(String baseUrl) {
 
     public int count() throws SearchEngineException {
         try {
-            String url = baseUrl + "/_doc/_count";
-            Response response = Request.Get(url).execute();
-            String json = response.returnContent().asString();
+            String url = baseUrl + "/_count";
+            HttpClient httpClient = ESHttpBasicClientFactory.getHttpClient(baseUrl);
+            HttpResponse response = httpClient.execute(new HttpGet(url));
+            String json = EntityUtils.toString(response.getEntity());
 
             @SuppressWarnings("unchecked")
             Map<String, Object> map = new ObjectMapper().readValue(json,