Skip to content

Commit 01ae7a5

Browse files
jmaeng72jmaeng72
committed
consolidate dup funcs into common util file
1 parent 6eab2a0 commit 01ae7a5

3 files changed

Lines changed: 81 additions & 106 deletions

File tree

‎elastic-utils-lib/src/cmr/elastic_utils/es_helper.clj‎

Lines changed: 25 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -6,47 +6,17 @@
66
[clojure.string :as string]
77
[cmr.common.services.errors :as errors]
88
[cmr.elastic-utils.config :as es-config]
9+
[cmr.elastic-utils.es-util :as es-util]
910
[cmr.transmit.config :as t-config]))
1011

11-
(defn- parse-safely
12-
"Parses the json body from the response safely"
13-
[body]
14-
(when body
15-
(if (string? body)
16-
(json/decode body true)
17-
body)))
18-
19-
(defn- decode-response
20-
"Decodes the response body from the given response"
21-
[response]
22-
(-> response
23-
:body
24-
parse-safely))
25-
26-
(defn- join-names
27-
"Joins names together with a comma"
28-
[names]
29-
(if (sequential? names)
30-
(string/join "," names)
31-
names))
32-
33-
(defn- url-with-path
34-
"Returns the url with the given path"
35-
[conn & path-parts]
36-
(let [path (->> path-parts
37-
(map join-names)
38-
(filter identity)
39-
(string/join "/"))]
40-
(str (:uri conn) "/" path)))
41-
4212
(defn search
4313
"Performs a search query across one or more indexes"
4414
[conn index _mapping-type opts]
4515
(let [qk [:search_type :scroll :routing :preference :ignore_unavailable]
4616
qp (merge {:track_total_hits true}
4717
(select-keys opts qk))
4818
body (apply dissoc opts qk)
49-
url (url-with-path conn index "_search")]
19+
url (es-util/url-with-path conn index "_search")]
5020
(let [response (http/post url
5121
(merge (:http-opts conn)
5222
{:content-type :json
@@ -56,14 +26,14 @@
5626
:throw-exceptions false}))
5727
status (:status response)]
5828
(if (some #{status} [200 201])
59-
(decode-response response)
29+
(es-util/decode-response response)
6030
(throw (ex-info (str "Search failed with status " status)
6131
{:status status :body (:body response)}))))))
6232

6333
(defn count-query
6434
"Performs a count query over one or more indexes"
6535
[conn index _mapping-type query]
66-
(let [url (url-with-path conn index "_count")
36+
(let [url (es-util/url-with-path conn index "_count")
6737
body (if (get query :query)
6838
query
6939
{:query query})]
@@ -75,17 +45,17 @@
7545
:throw-exceptions false}))
7646
status (:status response)]
7747
(if (some #{status} [200 201])
78-
(decode-response response)
48+
(es-util/decode-response response)
7949
(throw (ex-info (str "Count failed with status " status)
8050
{:status status :body (:body response)}))))))
8151

8252
(defn scroll
8353
"Performs a scroll query, fetching the next page of results from a query given a scroll id"
8454
[conn scroll-id opts]
85-
(let [url (url-with-path conn "_search" "scroll")
55+
(let [url (es-util/url-with-path conn "_search" "scroll")
8656
body (merge {:scroll_id scroll-id}
8757
(select-keys opts [:scroll]))]
88-
(decode-response
58+
(es-util/decode-response
8959
(http/post url
9060
(merge (:http-opts conn)
9161
{:content-type :json
@@ -97,23 +67,23 @@
9767
([conn index mapping-type id]
9868
(doc-get conn index mapping-type id nil))
9969
([conn index _mapping-type id opts]
100-
(let [url (url-with-path conn index "_doc" id)
70+
(let [url (es-util/url-with-path conn index "_doc" id)
10171
response (http/get url
10272
(merge (:http-opts conn)
10373
{:query-params opts
10474
:accept :json
10575
:throw-exceptions false}))
10676
status (:status response)]
10777
(when-not (= 404 status)
108-
(decode-response response)))))
78+
(es-util/decode-response response)))))
10979

11080
(defn put
11181
"Creates or updates a document in the search index, using the provided document id"
11282
([conn index mapping-type id document]
11383
(put conn index mapping-type id document nil))
11484
([conn index _mapping-type id document opts]
115-
(let [url (url-with-path conn index "_doc" id)]
116-
(decode-response
85+
(let [url (es-util/url-with-path conn index "_doc" id)]
86+
(es-util/decode-response
11787
(http/put url
11888
(merge (:http-opts conn)
11989
{:content-type :json
@@ -127,8 +97,8 @@
12797
([conn index mapping-type id]
12898
(delete conn index mapping-type id nil))
12999
([conn index _mapping-type id opts]
130-
(let [url (url-with-path conn index "_doc" id)]
131-
(decode-response
100+
(let [url (es-util/url-with-path conn index "_doc" id)]
101+
(es-util/decode-response
132102
(http/delete url
133103
(merge (:http-opts conn)
134104
{:content-type :json
@@ -142,7 +112,7 @@
142112
otherwise specifying a string suffices."
143113
[conn index _mapping-type query]
144114
(let [admin-token (es-config/elastic-admin-token)
145-
url (url-with-path conn index "_delete_by_query")
115+
url (es-util/url-with-path conn index "_delete_by_query")
146116
response (http/post url
147117
(merge (:http-opts conn)
148118
{:headers {"Authorization" admin-token
@@ -153,15 +123,15 @@
153123
:throw-exceptions false}))
154124
status (:status response)]
155125
(if (#{200 201} status)
156-
(decode-response response)
126+
(es-util/decode-response response)
157127
(throw (ex-info (str "Delete by query failed with status " status)
158128
{:status status :body (:body response)})))))
159129

160130
(defn delete-index
161131
"Deletes an index from the elastic store"
162132
[conn index]
163-
(let [url (url-with-path conn index)]
164-
(decode-response
133+
(let [url (es-util/url-with-path conn index)]
134+
(es-util/decode-response
165135
(http/delete url
166136
(merge (:http-opts conn)
167137
{:accept :json})))))
@@ -171,8 +141,8 @@
171141
([conn operations] (bulk conn operations nil))
172142
([conn operations params]
173143
(when (not-empty operations)
174-
(let [url (url-with-path conn "_bulk")]
175-
(decode-response
144+
(let [url (es-util/url-with-path conn "_bulk")]
145+
(es-util/decode-response
176146
(http/post url
177147
(merge (:http-opts conn)
178148
{:body (-> (map json/encode operations)
@@ -187,8 +157,8 @@
187157
(defn clear-scroll
188158
"Performs a clear scroll call for the given scroll id"
189159
[conn scroll-id]
190-
(let [url (url-with-path conn "_search" "scroll")]
191-
(decode-response
160+
(let [url (es-util/url-with-path conn "_search" "scroll")]
161+
(es-util/decode-response
192162
(http/delete url
193163
(merge (:http-opts conn)
194164
{:content-type :json
@@ -202,8 +172,8 @@
202172
(let [body {"source" {:index source-index}
203173
"dest" {:index target-index
204174
:version_type "external_gte"}}
205-
url (str (url-with-path conn "_reindex") "?wait_for_completion=false")]
206-
(decode-response
175+
url (str (es-util/url-with-path conn "_reindex") "?wait_for_completion=false")]
176+
(es-util/decode-response
207177
(http/post url
208178
(merge (:http-opts conn)
209179
{:body (json/encode body)
@@ -215,8 +185,8 @@
215185
Returns a map that captures the complete status and if there were any failures."
216186
[conn index reindex-task-id]
217187
(try
218-
(let [url (url-with-path conn "_tasks" reindex-task-id)
219-
resp (decode-response
188+
(let [url (es-util/url-with-path conn "_tasks" reindex-task-id)
189+
resp (es-util/decode-response
220190
(http/get url
221191
(merge (:http-opts conn)
222192
{:accept :json})))

‎elastic-utils-lib/src/cmr/elastic_utils/es_index_helper.clj‎

Lines changed: 20 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -3,40 +3,9 @@
33
(:require
44
[cheshire.core :as json]
55
[clj-http.client :as client]
6-
[clojure.string :as string]
6+
[cmr.elastic-utils.es-util :as es-util]
77
[cmr.transmit.config :as config]))
88

9-
(defn- parse-safely
10-
"Parses the json body from the response safely"
11-
[body]
12-
(when body
13-
(if (string? body)
14-
(json/decode body true)
15-
body)))
16-
17-
(defn- decode-response
18-
"Decodes the response body from the given response"
19-
[response]
20-
(-> response
21-
:body
22-
parse-safely))
23-
24-
(defn- join-names
25-
"Joins names together with a comma"
26-
[names]
27-
(if (sequential? names)
28-
(string/join "," names)
29-
names))
30-
31-
(defn- url-with-path
32-
"Returns the url with the given path"
33-
[conn & path-parts]
34-
(let [path (->> path-parts
35-
(map join-names)
36-
(filter identity)
37-
(string/join "/"))]
38-
(str (:uri conn) "/" path)))
39-
409
(defn index-alias
4110
"Returns the default index alias for the given index"
4211
[index-name]
@@ -45,15 +14,15 @@
4514
(defn exists?
4615
"Return true if the given index exists"
4716
[conn index-name]
48-
(let [url (url-with-path conn index-name)
17+
(let [url (es-util/url-with-path conn index-name)
4918
response (client/head url (merge (:http-opts conn) {:throw-exceptions false}))]
5019
(= 200 (:status response))))
5120

5221
(defn update-mapping
5322
"Register or modify specific mapping definition. Note that ES index mapping updates performs a MERGE and not a REPLACE. So properties are either added or changed, but never deleted."
5423
[conn index-name-or-names _type-name opts]
5524
(let [{:keys [mapping]} opts
56-
url (url-with-path conn index-name-or-names "_mapping")
25+
url (es-util/url-with-path conn index-name-or-names "_mapping")
5726
response (client/put url
5827
(merge (:http-opts conn)
5928
{:content-type :json
@@ -63,15 +32,15 @@
6332
:throw-exceptions false}))
6433
status (:status response)]
6534
(if (some #{status} [200 201])
66-
(decode-response response)
35+
(es-util/decode-response response)
6736
(throw (ex-info (str "Update mapping failed with status " status)
6837
{:status status :body (:body response)})))))
6938

7039
(defn create
7140
"Create an index"
7241
[conn index-name opts]
7342
(let [{:keys [settings mappings]} opts
74-
url (url-with-path conn index-name)
43+
url (es-util/url-with-path conn index-name)
7544
body (cond-> {:settings (or settings {})}
7645
mappings (assoc :mappings mappings))]
7746
(let [response (client/put url
@@ -83,15 +52,15 @@
8352
:throw-exceptions false}))
8453
status (:status response)]
8554
(if (some #{status} [200 201])
86-
(decode-response response)
55+
(es-util/decode-response response)
8756
(throw (ex-info (str "Create index failed with status " status)
8857
{:status status :body (:body response)}))))))
8958

9059
(defn refresh
9160
"Refresh an index"
9261
[conn index-name]
93-
(let [url (url-with-path conn index-name "_refresh")]
94-
(decode-response
62+
(let [url (es-util/url-with-path conn index-name "_refresh")]
63+
(es-util/decode-response
9564
(client/post url (merge (:http-opts conn)
9665
{:accept :json
9766
:content-type :json
@@ -100,16 +69,16 @@
10069
(defn delete
10170
"Delete an index"
10271
[conn index-name]
103-
(let [url (url-with-path conn index-name)]
104-
(decode-response
72+
(let [url (es-util/url-with-path conn index-name)]
73+
(es-util/decode-response
10574
(client/delete url (merge (:http-opts conn)
10675
{:accept :json})))))
10776

10877
(defn update-aliases
10978
"Update index aliases"
11079
[conn actions]
111-
(let [url (url-with-path conn "_aliases")]
112-
(decode-response
80+
(let [url (es-util/url-with-path conn "_aliases")]
81+
(es-util/decode-response
11382
(client/post url (merge (:http-opts conn)
11483
{:content-type :json
11584
:body (json/generate-string {:actions actions})
@@ -118,14 +87,14 @@
11887
(defn get-aliases
11988
"Get index aliases"
12089
[conn index-name]
121-
(let [url (url-with-path conn index-name "_alias")
90+
(let [url (es-util/url-with-path conn index-name "_alias")
12291
response (client/get url (merge (:http-opts conn)
12392
{:accept :json
12493
:throw-exceptions false}))
12594
status (:status response)]
12695
(if (= 404 status)
12796
[]
128-
(let [resp (decode-response response)
97+
(let [resp (es-util/decode-response response)
12998
aliases (keys (get-in resp [(keyword index-name) :aliases]))]
13099
(mapv name aliases)))))
131100

@@ -138,13 +107,13 @@
138107
"Create an index template in elasticsearch"
139108
[conn template-name opts]
140109
(let [{:keys [index-patterns settings mappings aliases]} opts
141-
url (url-with-path conn "_index_template" template-name)
110+
url (es-util/url-with-path conn "_index_template" template-name)
142111
template (merge {:settings settings}
143112
(when mappings {:mappings mappings})
144113
(when aliases {:aliases aliases}))
145114
body {:index_patterns index-patterns
146115
:template template}]
147-
(decode-response
116+
(es-util/decode-response
148117
(client/post url (merge (:http-opts conn)
149118
{:content-type :json
150119
:body (json/generate-string body)
@@ -153,17 +122,17 @@
153122
(defn get-mapping
154123
"Get the mapping for an index"
155124
[conn index-name]
156-
(let [url (url-with-path conn index-name "_mapping")]
157-
(decode-response
125+
(let [url (es-util/url-with-path conn index-name "_mapping")]
126+
(es-util/decode-response
158127
(client/get url (merge (:http-opts conn)
159128
{:accept :json
160129
:throw-exceptions false})))))
161130

162131
(defn get-settings
163132
"Get the settings for an index"
164133
[conn index-name]
165-
(let [url (url-with-path conn index-name "_settings")]
166-
(decode-response
134+
(let [url (es-util/url-with-path conn index-name "_settings")]
135+
(es-util/decode-response
167136
(client/get url (merge (:http-opts conn)
168137
{:accept :json
169138
:throw-exceptions false})))))

0 commit comments

Comments
 (0)