diff --git a/bootstrap-app/src/cmr/bootstrap/data/rebalance_util.clj b/bootstrap-app/src/cmr/bootstrap/data/rebalance_util.clj index 54acc214f4..6d4f83d156 100644 --- a/bootstrap-app/src/cmr/bootstrap/data/rebalance_util.clj +++ b/bootstrap-app/src/cmr/bootstrap/data/rebalance_util.clj @@ -1,7 +1,6 @@ (ns cmr.bootstrap.data.rebalance-util "Utilities for helping with rebalancing a collection." (:require - [clojurewerkz.elastisch.query :as q] [cmr.bootstrap.embedded-system-helper :as helper] [cmr.elastic-utils.es-helper :as es-helper] [cmr.elastic-utils.config :as es-config] @@ -15,8 +14,8 @@ (defn es-query-for-collection-concept-id "Returns an elasticsearch query to find granules in the collection." [concept-id] - {:bool {:must (q/match-all) - :filter (q/term :collection-concept-id concept-id)}}) + {:bool {:must {:match_all {}} + :filter {:term {:collection-concept-id concept-id}}}}) (defn- granule-count-for-collection "Gets the granule count for the collection in the elastic index." diff --git a/elastic-utils-lib/project.clj b/elastic-utils-lib/project.clj index cafa857f49..b1a703ce99 100644 --- a/elastic-utils-lib/project.clj +++ b/elastic-utils-lib/project.clj @@ -8,12 +8,12 @@ :dependencies [[cheshire :exclusions [com.fasterxml.jackson.core/jackson-core]] [clj-http] - [clojurewerkz/elastisch "5.0.0-beta1"] [commons-codec/commons-codec "1.11"] [commons-io "2.18.0"] [nasa-cmr/cmr-common-lib "0.1.1-SNAPSHOT"] [nasa-cmr/cmr-transmit-lib "0.1.0-SNAPSHOT"] [org.clojure/clojure] + [ring/ring-jetty-adapter "1.14.2"] ;; net.jpountz.lz4 and org.lz4 is no longer supported and at.yawk.lz4 is a drop in ;; replacement for it. Exclude these libraries to prevent conflicts. diff --git a/elastic-utils-lib/src/cmr/elastic_utils/connect.clj b/elastic-utils-lib/src/cmr/elastic_utils/connect.clj index b871369a21..b5c38ccfd8 100644 --- a/elastic-utils-lib/src/cmr/elastic_utils/connect.clj +++ b/elastic-utils-lib/src/cmr/elastic_utils/connect.clj @@ -1,15 +1,17 @@ (ns cmr.elastic-utils.connect "Provide functions to invoke elasticsearch" (:require + [cheshire.core :as json] + [clj-http.client :as client] [clj-http.conn-mgr :as conn-mgr] - [clojurewerkz.elastisch.rest :as esr] - [clojurewerkz.elastisch.rest.admin :as admin] [cmr.common.api.web-server :as web-server] [cmr.common.log :as log :refer (info)] [cmr.common.services.errors :as errors] [cmr.common.services.health-helper :as hh] [cmr.common.util :as util])) +(defrecord Connection [uri http-opts]) + (def ELASTIC_CONNECTION_TIMOUT "The number of milliseconds to wait before timing out a connection attempt to elasticsearch. Currently set to 5 minutes." @@ -41,7 +43,7 @@ (info (format "Connecting to single ES on %s %d using retry-handler %s" host port retry-handler)) - (esr/connect (str "http://" host ":" port) http-options))) + (->Connection (str "http://" host ":" port) http-options))) (defn try-connect [config] @@ -51,24 +53,30 @@ (errors/internal-error! (format "Unable to connect to elasticsearch at: %s. with %s" config e))))) +(defn- get-elastic-health + "Returns the elastic health by calling elasticsearch cluster health api" + ([conn] + (get-elastic-health conn {:wait_for_status "yellow" + :timeout (str (hh/health-check-timeout-seconds) "s")})) + ([conn params] + (try + (let [url (str (:uri conn) "/_cluster/health") + response (client/get url (merge (:http-opts conn) + {:query-params params + :accept :json}))] + (json/decode (:body response) true)) + (catch Exception e + (format "Unable to get elasticsearch cluster health, caught exception: %s" + (.getMessage e)))))) + (defn wait-for-healthy-elastic "Waits for the elasticsearch cluster health to reach yellow. Pass in a elasticsearch store that has a :conn key with the elastisch connection" [elastic-store] - (when (:timed_out (admin/cluster-health - (:conn elastic-store) {:wait_for_status "yellow" :timeout "3s"})) + (when (:timed_out (get-elastic-health (:conn elastic-store) + {:wait_for_status "yellow" :timeout "3s"})) (errors/internal-error! "Timed out waiting for elasticsearch to reach a healthy state"))) -(defn- get-elastic-health - "Returns the elastic health by calling elasticsearch cluster health api" - [conn] - (try - (admin/cluster-health conn {:wait_for_status "yellow" - :timeout (str (hh/health-check-timeout-seconds) "s")}) - (catch Exception e - (format "Unable to get elasticsearch cluster health, caught exception: %s" - (.getMessage e))))) - (defn health-fn "Returns the health state of elasticsearch." [context elastic-key-in-context] diff --git a/elastic-utils-lib/src/cmr/elastic_utils/es_helper.clj b/elastic-utils-lib/src/cmr/elastic_utils/es_helper.clj index ac9c43602d..a993709a98 100644 --- a/elastic-utils-lib/src/cmr/elastic_utils/es_helper.clj +++ b/elastic-utils-lib/src/cmr/elastic_utils/es_helper.clj @@ -4,13 +4,9 @@ [cheshire.core :as json] [clj-http.client :as http] [clojure.string :as string] - [clojurewerkz.elastisch.rest :as rest] - [clojurewerkz.elastisch.rest.document :as doc] - [clojurewerkz.elastisch.rest.response :refer [not-found?]] - [clojurewerkz.elastisch.rest.utils :refer [join-names]] - [cmr.common.log :refer [info]] [cmr.common.services.errors :as errors] [cmr.elastic-utils.config :as es-config] + [cmr.elastic-utils.es-util :as es-util] [cmr.transmit.config :as t-config])) (defn search @@ -20,9 +16,9 @@ qp (merge {:track_total_hits true} (select-keys opts qk)) body (apply dissoc opts qk) - url (format "%s/%s/_search" (:uri conn) (join-names index))] + url (es-util/url-with-path conn index "_search")] (let [response (http/post url - (merge (.http-opts conn) + (merge (:http-opts conn) {:content-type :json :body (json/generate-string body) :query-params qp @@ -30,113 +26,145 @@ :throw-exceptions false})) status (:status response)] (if (some #{status} [200 201]) - (rest/parse-safely (:body response)) + (es-util/decode-response response) (throw (ex-info (str "Search failed with status " status) {:status status :body (:body response)})))))) (defn count-query "Performs a count query over one or more indexes" [conn index _mapping-type query] - (let [url (format "%s/%s/_count" (:uri conn) (join-names index)) + (let [url (es-util/url-with-path conn index "_count") body (if (get query :query) query {:query query})] (let [response (http/post url - (merge (.http-opts conn) + (merge (:http-opts conn) {:content-type :json :body (json/generate-string body) :accept :json :throw-exceptions false})) status (:status response)] (if (some #{status} [200 201]) - (rest/parse-safely (:body response)) + (es-util/decode-response response) (throw (ex-info (str "Count failed with status " status) {:status status :body (:body response)})))))) (defn scroll "Performs a scroll query, fetching the next page of results from a query given a scroll id" [conn scroll-id opts] - (doc/scroll conn scroll-id opts)) + (let [url (es-util/url-with-path conn "_search" "scroll") + body (merge {:scroll_id scroll-id} + (select-keys opts [:scroll]))] + (es-util/decode-response + (http/post url + (merge (:http-opts conn) + {:content-type :json + :body (json/generate-string body) + :accept :json}))))) (defn doc-get "Fetches and returns a document by id or `nil` if it does not exist." ([conn index mapping-type id] (doc-get conn index mapping-type id nil)) ([conn index _mapping-type id opts] - (let [result (if (empty? opts) - (rest/get conn (rest/record-url conn index "_doc" id)) - (rest/get conn (rest/record-url conn index "_doc" id) {:query-params opts}))] - (when-not (not-found? result) - result)))) + (let [url (es-util/url-with-path conn index "_doc" id) + response (http/get url + (merge (:http-opts conn) + {:query-params opts + :accept :json + :throw-exceptions false})) + status (:status response)] + (when-not (= 404 status) + (es-util/decode-response response))))) (defn put "Creates or updates a document in the search index, using the provided document id" ([conn index mapping-type id document] (put conn index mapping-type id document nil)) ([conn index _mapping-type id document opts] - (rest/put conn (rest/record-url conn index "_doc" id) - {:content-type :json - :body document - :query-params opts}))) + (let [url (es-util/url-with-path conn index "_doc" id)] + (es-util/decode-response + (http/put url + (merge (:http-opts conn) + {:content-type :json + :body (if (string? document) document (json/generate-string document)) + :query-params opts + :accept :json + :throw-exceptions false})))))) (defn delete "Deletes document from the index." ([conn index mapping-type id] (delete conn index mapping-type id nil)) ([conn index _mapping-type id opts] - (-> (rest/record-url conn index "_doc" id) - (http/delete (merge {:throw-exceptions false} - (.http-opts conn) - {:content-type :json :query-params opts} - {:accept :json})) - (:body) - (rest/parse-safely)))) + (let [url (es-util/url-with-path conn index "_doc" id)] + (es-util/decode-response + (http/delete url + (merge (:http-opts conn) + {:content-type :json + :query-params opts + :accept :json + :throw-exceptions false})))))) (defn delete-by-query "Performs a delete-by-query operation over one or more indexes and types. Multiple indexes and types can be specified by passing in a seq of strings, otherwise specifying a string suffices." [conn index _mapping-type query] - (let [admin-token (es-config/elastic-admin-token) - delete-url (rest/delete-by-query-url - conn - (join-names index)) - response (http/post delete-url - {:headers {"Authorization" admin-token - "Confirm-delete-action" "true" - :client-id t-config/cmr-client-id} - :content-type :json - :body (json/generate-string {:query query}) - :throw-exceptions false}) - status (:status response)] - (if (#{200 201} status) - (rest/parse-safely (:body response)) - (throw (ex-info (str "Delete by query failed with status " status) - {:status status :body (:body response)}))))) + (let [admin-token (es-config/elastic-admin-token) + url (es-util/url-with-path conn index "_delete_by_query") + response (http/post url + (merge (:http-opts conn) + {:headers {"Authorization" admin-token + "Confirm-delete-action" "true" + :client-id t-config/cmr-client-id} + :content-type :json + :body (json/generate-string {:query query}) + :throw-exceptions false})) + status (:status response)] + (if (#{200 201} status) + (es-util/decode-response response) + (throw (ex-info (str "Delete by query failed with status " status) + {:status status :body (:body response)}))))) (defn delete-index "Deletes an index from the elastic store" [conn index] - (rest/delete conn - (rest/url-with-path conn index))) + (let [url (es-util/url-with-path conn index)] + (es-util/decode-response + (http/delete url + (merge (:http-opts conn) + {:accept :json}))))) (defn bulk "Performs a bulk operation" ([conn operations] (bulk conn operations nil)) ([conn operations params] (when (not-empty operations) - (rest/post-string conn (rest/bulk-url conn) - {:body (-> (map json/encode operations) - (interleave (repeat "\n")) - (string/join)) - :content-type "application/x-ndjson" - :query-params params})))) + (let [url (es-util/url-with-path conn "_bulk")] + (es-util/decode-response + (http/post url + (merge (:http-opts conn) + {:body (-> (map json/encode operations) + (interleave (repeat "\n")) + (string/join) + (str "\n")) + :content-type "application/x-ndjson" + :query-params params + :accept :json + :throw-exceptions false}))))))) (defn clear-scroll "Performs a clear scroll call for the given scroll id" [conn scroll-id] - (rest/delete conn (rest/scroll-url conn) {:content-type :json - :body {:scroll_id scroll-id}})) + (let [url (es-util/url-with-path conn "_search" "scroll")] + (es-util/decode-response + (http/delete url + (merge (:http-opts conn) + {:content-type :json + :body (json/generate-string {:scroll_id scroll-id}) + :accept :json + :throw-exceptions false}))))) (defn migrate-index "Copies the contents of one index into another. Used for resharding." @@ -144,40 +172,24 @@ (let [body {"source" {:index source-index} "dest" {:index target-index :version_type "external_gte"}} - url (str (rest/url-with-path conn "_reindex") "?wait_for_completion=false")] - (rest/post-string conn url - {:body (json/encode body) - :content-type "application/json"}))) + url (str (es-util/url-with-path conn "_reindex") "?wait_for_completion=false")] + (es-util/decode-response + (http/post url + (merge (:http-opts conn) + {:body (json/encode body) + :content-type "application/json" + :accept :json}))))) (defn get-reindex-task-status "Get the reindex task status and if there were any failures if the task is considered COMPLETE. - Returns a map that captures the complete status and if there were any failures. - Example map return: - { - :complete true - :failures [{ - :index: \"new_index\", - :type: \"_doc\", - :id\": \"doc_id\", - :cause: { - :type: \"exception\", - :reason: \"failure reason\" - :caused_by: { - :type: \"number_format_exception\", - :reason: \"cause reason\" - } - } - :status: 400 - }] - :error {:type \"type\" - :reason \"reason\" - :caused_by {} - } - }" + Returns a map that captures the complete status and if there were any failures." [conn index reindex-task-id] (try - (let [url (rest/url-with-path conn "_tasks" reindex-task-id) - resp (rest/get conn url) + (let [url (es-util/url-with-path conn "_tasks" reindex-task-id) + resp (es-util/decode-response + (http/get url + (merge (:http-opts conn) + {:accept :json}))) completed (:completed resp) failures (get-in resp [:response :failures]) task-error (:error resp) diff --git a/elastic-utils-lib/src/cmr/elastic_utils/es_index_helper.clj b/elastic-utils-lib/src/cmr/elastic_utils/es_index_helper.clj index bc50b952c2..40d3f66943 100644 --- a/elastic-utils-lib/src/cmr/elastic_utils/es_index_helper.clj +++ b/elastic-utils-lib/src/cmr/elastic_utils/es_index_helper.clj @@ -3,12 +3,8 @@ (:require [cheshire.core :as json] [clj-http.client :as client] - [clojurewerkz.elastisch.rest :as rest] - [clojurewerkz.elastisch.rest.index :as esi] - [clojurewerkz.elastisch.rest.utils :refer [join-names]] - [cmr.transmit.config :as config]) - #_{:clj-kondo/ignore [:unused-import]} - (:import clojurewerkz.elastisch.rest.Connection)) + [cmr.elastic-utils.es-util :as es-util] + [cmr.transmit.config :as config])) (defn index-alias "Returns the default index alias for the given index" @@ -18,15 +14,17 @@ (defn exists? "Return true if the given index exists" [conn index-name] - (esi/exists? conn index-name)) + (let [url (es-util/url-with-path conn index-name) + response (client/head url (merge (:http-opts conn) {:throw-exceptions false}))] + (= 200 (:status response)))) (defn update-mapping "Register or modify specific mapping definition. Note that ES index mapping updates performs a MERGE and not a REPLACE. So properties are either added or changed, but never deleted." [conn index-name-or-names _type-name opts] (let [{:keys [mapping]} opts - url (format "%s/%s/_mapping" (:uri conn) (join-names index-name-or-names)) + url (es-util/url-with-path conn index-name-or-names "_mapping") response (client/put url - (merge (.http-opts conn) + (merge (:http-opts conn) {:content-type :json :body (json/generate-string mapping) :query-params (dissoc opts :mapping) @@ -34,7 +32,7 @@ :throw-exceptions false})) status (:status response)] (if (some #{status} [200 201]) - (rest/parse-safely (:body response)) + (es-util/decode-response response) (throw (ex-info (str "Update mapping failed with status " status) {:status status :body (:body response)}))))) @@ -42,11 +40,11 @@ "Create an index" [conn index-name opts] (let [{:keys [settings mappings]} opts - url (format "%s/%s" (:uri conn) index-name) + url (es-util/url-with-path conn index-name) body (cond-> {:settings (or settings {})} mappings (assoc :mappings mappings))] (let [response (client/put url - (merge (.http-opts conn) + (merge (:http-opts conn) {:content-type :json :body (json/generate-string body) :query-params (dissoc opts :mappings :settings) @@ -54,43 +52,51 @@ :throw-exceptions false})) status (:status response)] (if (some #{status} [200 201]) - (rest/parse-safely (:body response)) + (es-util/decode-response response) (throw (ex-info (str "Create index failed with status " status) {:status status :body (:body response)})))))) (defn refresh "Refresh an index" [conn index-name] - (-> (rest/index-refresh-url conn (join-names index-name)) - (client/post (merge (.http-opts conn) - {:accept :json - :content-type :json - :headers {:client-id config/cmr-client-id}})) - (:body) - (rest/parse-safely))) + (let [url (es-util/url-with-path conn index-name "_refresh")] + (es-util/decode-response + (client/post url (merge (:http-opts conn) + {:accept :json + :content-type :json + :headers {:client-id config/cmr-client-id}}))))) (defn delete "Delete an index" [conn index-name] - (esi/delete conn index-name)) + (let [url (es-util/url-with-path conn index-name)] + (es-util/decode-response + (client/delete url (merge (:http-opts conn) + {:accept :json}))))) (defn update-aliases "Update index aliases" [conn actions] - (rest/post conn - (rest/index-aliases-batch-url conn) - {:content-type :json - :body {:actions actions}})) + (let [url (es-util/url-with-path conn "_aliases")] + (es-util/decode-response + (client/post url (merge (:http-opts conn) + {:content-type :json + :body (json/generate-string {:actions actions}) + :accept :json}))))) -;; We have to roll our own get-aliases function because Elasticsearch route on GET alias -;; for an index has changed and clojurewerkz is outdated (defn get-aliases "Get index aliases" [conn index-name] - (let [aliases-url (rest/url-with-path conn index-name "_alias") - resp (rest/get conn aliases-url) - aliases (keys (get-in resp [(keyword index-name) :aliases]))] - (mapv name aliases))) + (let [url (es-util/url-with-path conn index-name "_alias") + response (client/get url (merge (:http-opts conn) + {:accept :json + :throw-exceptions false})) + status (:status response)] + (if (= 404 status) + [] + (let [resp (es-util/decode-response response) + aliases (keys (get-in resp [(keyword index-name) :aliases]))] + (mapv name aliases))))) (defn alias-exists? "Return true if the given index has the default alias in the form of _alias" @@ -101,24 +107,32 @@ "Create an index template in elasticsearch" [conn template-name opts] (let [{:keys [index-patterns settings mappings aliases]} opts - template-url (rest/url-with-path conn "_index_template" template-name) + url (es-util/url-with-path conn "_index_template" template-name) template (merge {:settings settings} (when mappings {:mappings mappings}) (when aliases {:aliases aliases})) body {:index_patterns index-patterns :template template}] - (rest/post conn template-url - {:content-type :json - :body body}))) + (es-util/decode-response + (client/post url (merge (:http-opts conn) + {:content-type :json + :body (json/generate-string body) + :accept :json}))))) (defn get-mapping "Get the mapping for an index" [conn index-name] - (let [url (rest/url-with-path conn index-name "_mapping")] - (rest/get conn url))) + (let [url (es-util/url-with-path conn index-name "_mapping")] + (es-util/decode-response + (client/get url (merge (:http-opts conn) + {:accept :json + :throw-exceptions false}))))) (defn get-settings "Get the settings for an index" [conn index-name] - (let [url (rest/url-with-path conn index-name "_settings")] - (rest/get conn url))) \ No newline at end of file + (let [url (es-util/url-with-path conn index-name "_settings")] + (es-util/decode-response + (client/get url (merge (:http-opts conn) + {:accept :json + :throw-exceptions false}))))) \ No newline at end of file diff --git a/elastic-utils-lib/src/cmr/elastic_utils/es_util.clj b/elastic-utils-lib/src/cmr/elastic_utils/es_util.clj new file mode 100644 index 0000000000..bce87a9bf7 --- /dev/null +++ b/elastic-utils-lib/src/cmr/elastic_utils/es_util.clj @@ -0,0 +1,36 @@ +(ns cmr.elastic-utils.es-util + "Defines shared funcs to create es helper functionality." + (:require + [cheshire.core :as json] + [clojure.string :as string])) + +(defn parse-safely + "Parses the json body from the response safely" + [body] + (when body + (if (string? body) + (json/decode body true) + body))) + +(defn decode-response + "Decodes the response body from the given response" + [response] + (-> response + :body + parse-safely)) + +(defn join-names + "Joins names together with a comma" + [names] + (if (sequential? names) + (string/join "," names) + names)) + +(defn url-with-path + "Returns the url with the given path" + [conn & path-parts] + (let [path (->> path-parts + (map join-names) + (filter identity) + (string/join "/"))] + (str (:uri conn) "/" path))) \ No newline at end of file diff --git a/elastic-utils-lib/src/cmr/elastic_utils/index_util.clj b/elastic-utils-lib/src/cmr/elastic_utils/index_util.clj index ab4ba0c45a..e5677e1cb5 100644 --- a/elastic-utils-lib/src/cmr/elastic_utils/index_util.clj +++ b/elastic-utils-lib/src/cmr/elastic_utils/index_util.clj @@ -1,6 +1,7 @@ (ns cmr.elastic-utils.index-util "Defines different types and functions for defining mappings" (:require + [cheshire.core :as json] [cmr.common.date-time-parser :as time-parser] [cmr.common.log :as log :refer (info)] [cmr.common.services.errors :as errors] @@ -191,9 +192,10 @@ `(try ~@body (catch clojure.lang.ExceptionInfo e# - (errors/internal-error! - (str "Call to Elasticsearch caught exception " (get-in (ex-data e#) [:object :body])) - e#)))) + (let [body# (get-in (ex-data e#) [:body]) + status# (:status (ex-data e#)) + parsed-body# (if (string? body#) (json/decode body# true) body#)] + {:error parsed-body# :status status#})))) (defn reset "Development time helper function to delete an index and recreate it to empty all data." diff --git a/elastic-utils-lib/src/cmr/elastic_utils/search/es_debug.clj b/elastic-utils-lib/src/cmr/elastic_utils/search/es_debug.clj index e8868ac6c4..c4b06a4489 100644 --- a/elastic-utils-lib/src/cmr/elastic_utils/search/es_debug.clj +++ b/elastic-utils-lib/src/cmr/elastic_utils/search/es_debug.clj @@ -1,10 +1,9 @@ (ns cmr.elastic-utils.search.es-debug - "Holds a very strange function which is only used by dev-system/control. Moved to this namespace - so as to not require any other files from needing to import clojurewerkz." + "Holds a very strange function which is only used by dev-system/control." (:require - [clojurewerkz.elastisch.rest.document :as esd] [cmr.common.services.errors :as e] [cmr.elastic-utils.config :as es-config] + [cmr.elastic-utils.es-helper :as es-helper] [cmr.elastic-utils.search.es-index :as common-esi] [cmr.elastic-utils.search.es-wrapper :as q])) @@ -23,12 +22,12 @@ Originally found in cmr.search.data.elastic-search-index/elastic_search_index.clj" [context] (let [index-info (common-esi/concept-type->index-info context :collection nil) - results (esd/search (context->conn context es-config/elastic-name) - (:index-name index-info) - [(:type-name index-info)] - :query (q/match-all) - :size 10000 - :_source ["permitted-group-ids"]) + results (es-helper/search (context->conn context es-config/elastic-name) + (:index-name index-info) + (:type-name index-info) + {:query (q/match-all) + :size 10000 + :_source ["permitted-group-ids"]}) hits (get-in results [:hits :total :value])] (when (> hits (count (get-in results [:hits :hits]))) (e/internal-error! "Failed to retrieve all hits.")) diff --git a/elastic-utils-lib/src/cmr/elastic_utils/search/es_index.clj b/elastic-utils-lib/src/cmr/elastic_utils/search/es_index.clj index 3b16f07563..0da78cd028 100644 --- a/elastic-utils-lib/src/cmr/elastic_utils/search/es_index.clj +++ b/elastic-utils-lib/src/cmr/elastic_utils/search/es_index.clj @@ -3,7 +3,6 @@ (:require [cheshire.core :as json] [clojure.set :as set] - [clojurewerkz.elastisch.rest.index :as esri] [cmr.common.concepts :as concepts] [cmr.common.lifecycle :as lifecycle] [cmr.common.log :refer [debug info warnf]] @@ -405,8 +404,8 @@ "Make changes written to Elasticsearch available for search. See https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-refresh.html" [context] - (esri/refresh (context->conn context es-config/elastic-name)) - (esri/refresh (context->conn context es-config/gran-elastic-name))) + (esi-helper/refresh (context->conn context es-config/elastic-name) nil) + (esi-helper/refresh (context->conn context es-config/gran-elastic-name) nil)) (defrecord ElasticSearchIndex ;; conn is the connection to elastic diff --git a/elastic-utils-lib/src/cmr/elastic_utils/search/es_query_to_elastic.clj b/elastic-utils-lib/src/cmr/elastic_utils/search/es_query_to_elastic.clj index 09574396c7..5401f739ec 100644 --- a/elastic-utils-lib/src/cmr/elastic_utils/search/es_query_to_elastic.clj +++ b/elastic-utils-lib/src/cmr/elastic_utils/search/es_query_to_elastic.clj @@ -3,7 +3,6 @@ NOTE: this originally lived at cmr.common-app.services.search.query-to-elastic" (:require [clojure.string :as string] - [clojurewerkz.elastisch.query :as query] [cmr.common.services.errors :as errors] [cmr.common.services.search.query-model :as qm] [cmr.elastic-utils.config :as config] @@ -74,7 +73,7 @@ [query] (let [{:keys [concept-type condition]} (query-expense/order-conditions query) core-query (condition->elastic condition concept-type)] - {:query {:bool {:must (query/match-all) + {:query {:bool {:must {:match_all {}} :filter core-query}}})) (defmethod query->elastic :autocomplete diff --git a/elastic-utils-lib/src/cmr/elastic_utils/search/es_wrapper.clj b/elastic-utils-lib/src/cmr/elastic_utils/search/es_wrapper.clj index 8af4b7aad4..590ebea92e 100644 --- a/elastic-utils-lib/src/cmr/elastic_utils/search/es_wrapper.clj +++ b/elastic-utils-lib/src/cmr/elastic_utils/search/es_wrapper.clj @@ -1,10 +1,7 @@ (ns cmr.elastic-utils.search.es-wrapper - "Wraps common elastic functions for use outside of elastic-utils so that other - namespaces do not need to import anything from clojurewerkz." - (:require - [clojurewerkz.elastisch.query :as query])) + "Wraps common elastic functions for use outside of elastic-utils.") (defn match-all - "See clojurewerkz for details" - ([] (query/match-all)) - ([opts] (query/match-all opts))) + "Returns a match-all query" + ([] {:match_all {}}) + ([opts] {:match_all opts})) diff --git a/indexer-app/int-test/cmr/indexer/test/services/index_fields.clj b/indexer-app/int-test/cmr/indexer/test/services/index_fields.clj index f4c656e61c..0393bd7b2f 100644 --- a/indexer-app/int-test/cmr/indexer/test/services/index_fields.clj +++ b/indexer-app/int-test/cmr/indexer/test/services/index_fields.clj @@ -5,14 +5,14 @@ [clojure.data.codec.base64 :as b64] [clojure.string :as string] [clojure.test :refer :all] - [clojurewerkz.elastisch.rest :as esr] [cmr.common.cache :as cache] [cmr.common.lifecycle :as lifecycle] [cmr.common.test.test-util :as tu] [cmr.elastic-utils.config :as es-config] + [cmr.elastic-utils.connect :as es] [cmr.elastic-utils.embedded-elastic-server :as elastic-server] [cmr.elastic-utils.es-index-helper :as esi] - [cmr.indexer.data.elasticsearch :as es] + [cmr.indexer.data.elasticsearch :as es-data] [cmr.indexer.data.index-set :as idx-set])) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -41,7 +41,7 @@ (save-document-in-elastic ["tests"] es-type es-doc concept-id revision-id options)) ([es-index es-type es-doc concept-id revision-id options] - (es/save-document-in-elastic + (es-data/save-document-in-elastic @context es-index es-type es-doc concept-id revision-id revision-id options))) @@ -52,14 +52,14 @@ (delete-document-in-elastic ["tests"] es-type concept-id revision-id options)) ([es-index es-type concept-id revision-id options] - (es/delete-document + (es-data/delete-document @context es-index es-type concept-id revision-id revision-id options))) (defn- get-document ([es-type concept-id] (get-document "tests" es-type concept-id)) ([es-index es-type concept-id] - (es/get-document @context es-index es-type concept-id))) + (es-data/get-document @context es-index es-type concept-id))) (defn- assert-same "Assert the retrieved document for the given concept and field has the @@ -110,9 +110,9 @@ (do (reset! context {:system {:gran-elastic {:config gran-elastic-test-config - :conn (esr/connect (str "http://localhost:" (:port gran-elastic-test-config)))} + :conn (es/try-connect gran-elastic-test-config)} :elastic {:config elastic-test-config - :conn (esr/connect (str "http://localhost:" (:port elastic-test-config)))}}}) + :conn (es/try-connect elastic-test-config)}}}) (try (f)))) diff --git a/indexer-app/int-test/cmr/indexer/test/utility.clj b/indexer-app/int-test/cmr/indexer/test/utility.clj index e3ae0909c2..c868a77e82 100644 --- a/indexer-app/int-test/cmr/indexer/test/utility.clj +++ b/indexer-app/int-test/cmr/indexer/test/utility.clj @@ -4,8 +4,8 @@ [cheshire.core :as cheshire] [clj-http.client :as client] [clojure.test :refer :all] - [clojurewerkz.elastisch.rest :as esr] [cmr.elastic-utils.config :as es-config] + [cmr.elastic-utils.connect :as es] [cmr.transmit.config :as transmit-config])) (defn indexer-root-url @@ -572,7 +572,7 @@ (defn reset-fixture [f] (reset) - (reset! gran-elastic-connection (esr/connect (gran-elastic-root))) - (reset! elastic-connection (esr/connect (elastic-root))) + (reset! gran-elastic-connection (es/try-connect {:host (es-config/gran-elastic-host) :port (es-config/gran-elastic-port)})) + (reset! elastic-connection (es/try-connect {:host (es-config/elastic-host) :port (es-config/elastic-port)})) (f) (reset)) diff --git a/indexer-app/int-test/cmr/indexer/test/valid_data_crud_tests.clj b/indexer-app/int-test/cmr/indexer/test/valid_data_crud_tests.clj index 5f561c146f..f5f99af388 100644 --- a/indexer-app/int-test/cmr/indexer/test/valid_data_crud_tests.clj +++ b/indexer-app/int-test/cmr/indexer/test/valid_data_crud_tests.clj @@ -3,7 +3,6 @@ (:require [clojure.string :as string] [clojure.test :refer [deftest is testing use-fixtures]] - [clojurewerkz.elastisch.rest.index :as esi] [cmr.elastic-utils.config :as es-config] [cmr.elastic-utils.es-index-helper :as esi-helper] [cmr.indexer.services.index-set-service :as svc] @@ -24,12 +23,12 @@ index-names-from-gran-cluster (svc/get-index-names index-set es-config/gran-elastic-name) index-names-from-non-gran-cluster (svc/get-index-names index-set es-config/elastic-name)] (doseq [idx-name index-names-from-gran-cluster] - (is (esi/exists? @util/gran-elastic-connection idx-name)) - (is (not (esi/exists? @util/elastic-connection idx-name))) + (is (esi-helper/exists? @util/gran-elastic-connection idx-name)) + (is (not (esi-helper/exists? @util/elastic-connection idx-name))) (is (= [(str idx-name "_alias")] (esi-helper/get-aliases @util/gran-elastic-connection idx-name)))) (doseq [idx-name index-names-from-non-gran-cluster] - (is (esi/exists? @util/elastic-connection idx-name)) - (is (not (esi/exists? @util/gran-elastic-connection idx-name))) + (is (esi-helper/exists? @util/elastic-connection idx-name)) + (is (not (esi-helper/exists? @util/gran-elastic-connection idx-name))) (is (= [(str idx-name "_alias")] (esi-helper/get-aliases @util/elastic-connection idx-name)))))) (testing "index-set doc existence" (let [index-set util/sample-index-set @@ -71,20 +70,20 @@ (is (= 201 status)) ;; check that coll idx is in non-gran cluster only - (is (esi/exists? @util/elastic-connection coll-idx-name)) - (is (not (esi/exists? @util/gran-elastic-connection coll-idx-name))) + (is (esi-helper/exists? @util/elastic-connection coll-idx-name)) + (is (not (esi-helper/exists? @util/gran-elastic-connection coll-idx-name))) ;; check that gran idx is in gran cluster only - (is (esi/exists? @util/gran-elastic-connection gran-idx-name)) - (is (not (esi/exists? @util/elastic-connection gran-idx-name))))) + (is (esi-helper/exists? @util/gran-elastic-connection gran-idx-name)) + (is (not (esi-helper/exists? @util/elastic-connection gran-idx-name))))) (testing "delete index-set" (let [{:keys [status]} (util/delete-index-set index-set-id) _ (is (= 204 status)) {:keys [status response]} (util/get-index-set index-set-id) _ (is (= 404 status))] ;; indices should be removed from their respective clusters - (is (not (esi/exists? @util/elastic-connection coll-idx-name))) - (is (not (esi/exists? @util/gran-elastic-connection gran-idx-name))))))) + (is (not (esi-helper/exists? @util/elastic-connection coll-idx-name))) + (is (not (esi-helper/exists? @util/gran-elastic-connection gran-idx-name))))))) ;; Verify get index-sets fetches all index-sets in elastic. ;; Create 2 index-sets with different ids but with same number of concepts and indices associated @@ -104,8 +103,8 @@ body (-> (util/get-index-sets) :response :body) actual-es-indices (util/list-es-indices body)] (doseq [es-idx-name actual-es-indices] - (is (or (esi/exists? @util/gran-elastic-connection es-idx-name) - (esi/exists? @util/elastic-connection es-idx-name)))) + (is (or (esi-helper/exists? @util/gran-elastic-connection es-idx-name) + (esi-helper/exists? @util/elastic-connection es-idx-name)))) (is (= expected-idx-cnt (count actual-es-indices)))))) ;; Verify that you can update an index set multiple times and get the correct indices created and deleted @@ -172,7 +171,7 @@ (doseq [collection expected-coll-indexes :let [collection-index-part (-> collection (string/replace "-" "_") string/lower-case) elastic-index-name (str util/sample-index-set-id "_" collection-index-part)]] - (is (esi/exists? @util/gran-elastic-connection elastic-index-name)) + (is (esi-helper/exists? @util/gran-elastic-connection elastic-index-name)) (is (esi-helper/alias-exists? @util/gran-elastic-connection elastic-index-name)))))) ;; Tests adding a collection that is rebalancing its granules from small_collections to a separate diff --git a/indexer-app/src/cmr/indexer/data/collection_granule_aggregation_cache.clj b/indexer-app/src/cmr/indexer/data/collection_granule_aggregation_cache.clj index 4072d08485..9d47242002 100644 --- a/indexer-app/src/cmr/indexer/data/collection_granule_aggregation_cache.clj +++ b/indexer-app/src/cmr/indexer/data/collection_granule_aggregation_cache.clj @@ -7,7 +7,6 @@ [clj-time.coerce :as tc] [clj-time.core :as t] [clj-time.format :as f] - [clojurewerkz.elastisch.query :as esq] [cmr.elastic-utils.datetime-helper :as datetime-helper] [cmr.common.cache :as c] [cmr.common.cache.fallback-cache :as fallback-cache] @@ -107,7 +106,7 @@ (-> (es-helper/search (indexer-util/context->conn context es-config/gran-elastic-name) "1_small_collections_alias,1_c*_alias" ;; Searching all granule indexes ["granule"] ;; With the granule type. - {:query (esq/match-all) + {:query {:match_all {}} :size 0 :aggs collection-aggregations}) parse-aggregations)) @@ -122,7 +121,7 @@ (-> (es-helper/search (indexer-util/context->conn context es-config/gran-elastic-name) "1_small_collections_alias,1_c*_alias" ;; Searching all granule indexes ["granule"] ;; With the granule type. - {:query {:bool {:must (esq/match-all) + {:query {:bool {:must {:match_all {}} :filter {:range {:revision-date-doc-values {:gte revision-date-str}}}}} :size 0 diff --git a/indexer-app/src/cmr/indexer/data/elasticsearch.clj b/indexer-app/src/cmr/indexer/data/elasticsearch.clj index 60f9a2ef77..0aa31a02f1 100644 --- a/indexer-app/src/cmr/indexer/data/elasticsearch.clj +++ b/indexer-app/src/cmr/indexer/data/elasticsearch.clj @@ -1,5 +1,6 @@ (ns cmr.indexer.data.elasticsearch (:require + [cheshire.core :as json] [clj-http.client :as client] [clojure.string :as string] [cmr.common.concepts :as cs] @@ -320,9 +321,10 @@ (try (f conn es-index es-type elastic-id es-doc options) (catch clojure.lang.ExceptionInfo e - (let [err-msg (get-in (ex-data e) [:body]) - msg (str "Call to Elasticsearch caught exception " err-msg)] - (errors/internal-error! msg)))))) + (let [body (get-in (ex-data e) [:body]) + status (:status (ex-data e)) + parsed-body (if (string? body) (json/decode body true) body)] + {:error parsed-body :status status}))))) (defn- context->es-config "Returns the elastic config in the context" diff --git a/indexer-app/src/cmr/indexer/data/index_set_elasticsearch.clj b/indexer-app/src/cmr/indexer/data/index_set_elasticsearch.clj index 7200477774..1a35baedf9 100644 --- a/indexer-app/src/cmr/indexer/data/index_set_elasticsearch.clj +++ b/indexer-app/src/cmr/indexer/data/index_set_elasticsearch.clj @@ -2,7 +2,6 @@ (:require [cheshire.core :as cheshire] [clj-http.client :as client] - [clojurewerkz.elastisch.rest :as esr] [cmr.common.log :as log :refer [info warn error]] [cmr.common.services.errors :as errors] [cmr.common.util :as util] @@ -149,7 +148,7 @@ [{:keys [conn config]} index-name] (when (esi-helper/exists? conn index-name) (let [admin-token (:admin-token config) - response (client/delete (esr/index-url conn index-name) + response (client/delete (str (:uri conn) "/" index-name) {:headers {"Authorization" admin-token "Confirm-delete-action" "true" :client-id t-config/cmr-client-id} diff --git a/system-int-test/test/cmr/system_int_test/search/misc/autocomplete_collection_facet_search_test.clj b/system-int-test/test/cmr/system_int_test/search/misc/autocomplete_collection_facet_search_test.clj index a0d92f57aa..9b0d08604b 100644 --- a/system-int-test/test/cmr/system_int_test/search/misc/autocomplete_collection_facet_search_test.clj +++ b/system-int-test/test/cmr/system_int_test/search/misc/autocomplete_collection_facet_search_test.clj @@ -3,8 +3,7 @@ (:require [clojure.test :refer :all] [cheshire.core :as json] - [clojurewerkz.elastisch.rest :as esr] - [clojurewerkz.elastisch.rest.document :as esd] + [clj-http.client :as client] [cmr.common.log :as log :refer [debug]] [cmr.common.util :refer [are3]] [cmr.elastic-utils.config :as es-config] @@ -41,8 +40,11 @@ (defn autocomplete-fixture [f] - (let [conn (esr/connect (url/elastic-root es-config/elastic-name)) - documents (map #(esd/create conn "1_autocomplete" "_doc" %) test-values)] + (let [root (url/elastic-root es-config/elastic-name) + url (str root "/1_autocomplete/_doc") + documents (map #(client/post url {:body (json/generate-string %) + :content-type :json + :accept :json}) test-values)] (doseq [doc documents] (debug "ingested " doc)) (index/wait-until-indexed) (f)