CMR-10569: Fixing search index cache issue.

Author: eereiter

elastic-utils-lib/src/cmr/elastic_utils/search/es_index_name_cache.clj

@@ -56,9 +56,15 @@
     {:index-names index-names-map
      :rebalancing-collections moving-to-separate-index}))
 
+(comment 
+ (println context3)
+  (fetch-concept-type-index-names context3)
+  (indexer/get-index-set context3 index-set-id)
+)
 (defn refresh-index-names-cache
   "Refresh the search index-names cache."
   [context]
+  (def context3 context)
   (info "Refreshing search index-names cache.")
   (let [index-names-map (fetch-concept-type-index-names context)
         index-names (:index-names index-names-map)

indexer-app/src/cmr/indexer/data/index_set.clj

@@ -7,10 +7,12 @@
    [cmr.common.concepts :as cs]
    [cmr.common.config :as cfg :refer [defconfig]]
    [cmr.common.generics :as common-generic]
+   [cmr.common.hash-cache :as hcache]
    [cmr.common.log :as log :refer (info error)]
    [cmr.common.util :as util]
    [cmr.common.services.errors :as errors]
    [cmr.elastic-utils.index-util :as m :refer [defmapping defnestedmapping]]
+   [cmr.elastic-utils.search.es-index-name-cache :as elastic-search-index-names-cache]
    [cmr.indexer.data.index-set-generics :as index-set-gen]
    [cmr.indexer.data.index-set-elasticsearch :as index-set-es]
    [cmr.indexer.services.index-set-service :as index-set-service]
@@ -1103,8 +1105,15 @@
   :indexer-index-set-cache)
 
 (comment
+  (let [cache (cache/context->cache context1 index-set-cache-key)]
+    (cache/get-keys cache))
+  (let [cache (cache/context->cache context1 index-set-cache-key)]
+    (cache/get-value cache :concept-mapping-types))
+  (let [cache (cache/context->cache context1 index-set-cache-key)]
+    (cache/get-value cache :concept-indices))
   (let [cache (cache/context->cache context index-set-cache-key)]
     (cache/get-keys cache))
+  
   (let [cache (cache/context->cache context index-set-cache-key)]
     (cache/get-value cache :concept-mapping-types))
 
@@ -1122,9 +1131,9 @@
     (cache/get-value cache :concept-indices (partial fetch-concept-type-index-names context))))
 
 (defn set-concept-type-index-names
-  "Fetch index names associated with concepts."
+  "Set index names associated with concepts."
   [context value]
-  (def context1 context)
+  (def context2 context)
   (let [cache (cache/context->cache context index-set-cache-key)]
     (cache/set-value cache :concept-indices value)))
 
@@ -1167,23 +1176,26 @@
 
 ;    (index-requested-index-set context index-set)))
 
-(comment
-  (let [index-set (index-set-es/get-index-set context index-set-id)]
-    (get-in index-set [:index-set :granule])
-    ;(cmr.indexer.services.index-set-service/update-index-set context index-set)
+(defn set-search-index-names-cache
+  ""
+  [context index-kw index-name]
+  (let [cache (hcache/context->cache context elastic-search-index-names-cache/index-names-cache-key)
+        granule (hcache/get-value cache elastic-search-index-names-cache/index-names-cache-key "granule")
+        new-granule (assoc granule index-kw index-name)]
+    (hcache/set-value cache elastic-search-index-names-cache/index-names-cache-key "granule" new-granule)))
+
+(comment 
+  (println coll-concept-id)
+  (let [coll-concept-id "C12323-PROV2"
+        {:keys [index-names rebalancing-collections]} (get-concept-type-index-names context)
+        indexes (:granule index-names)
+        provider (util/safe-lowercase (cs/concept-id->provider-id coll-concept-id))
+        idx-name (str "granules_" provider)
+        idx (get indexes (keyword idx-name))]
+    idx
     )
-  
-  (println target-index-key)
-  update-index-set
-  (get-granule-index-names-for-collection context "C1200000001-PROV2" nil)
-  (let [index-names (get-concept-type-index-names context)]
-    (get-in index-names [:index-names :granule]))
-  (let [{:keys [index-names rebalancing-collections]} (get-concept-type-index-names context)
-        indexes (:granule index-names)]
-    indexes)
-   ; (get indexes :small_collections))
-  (get-granule-index-names-for-provider context "prov6")
- )
+
+  )
 (defn get-granule-index-names-for-collection
   "Return the granule index names for the input collection concept id. Optionally a
    target-index-key can be specified which indicates that a specific index should be returned"
@@ -1200,26 +1212,34 @@
                                         (let [provider (util/safe-lowercase (cs/concept-id->provider-id coll-concept-id))
                                               idx-name (str "granules_" provider)
                                               idx (get indexes (keyword idx-name))]
-                                          ;(println "idx:" idx)
                                           (if idx
                                             idx
-                                            (let [index-set (index-set-es/get-index-set context index-set-id)
+                                            (let [_ (println "idx-name:" idx-name)
+                                                  _ (println "indexes:" indexes)
+                                                  _ (println "coll-concept-id:" coll-concept-id)
+                                                  index-set (index-set-es/get-index-set context index-set-id)
                                                   ;; This sets the index names in ES index.
-                                                  new-index-set (try 
-                                                                  (let [new-idx-set (add-new-granule-provider-index index-set idx-name)]
-                                                                    (index-set-service/update-index-set context new-idx-set))
-                                                                  (catch Exception e
-                                                                    (println (format "index %s already exists." idx-name))
-                                                                    index-set))
+                                                  new-index-set (let [new-idx-set (add-new-granule-provider-index index-set idx-name)]
+                                                                  (try
+                                                                    (index-set-service/update-index-set context new-idx-set)
+                                                                    (catch Exception _e
+                                                                      (println (format "index %s already exists." idx-name))
+                                                                      new-idx-set)))
                                                   _ (println "new-index-set" new-index-set)
+                                                  _ (when (nil? new-index-set)
+                                                      (println "index-set is:" index-set)
+                                                      (println "index-set is:" (:index-set index-set))
+                                                      (println "index-set is granule:" (:granule (:index-set index-set)))
+                                                      (println "index-set is:" (:indexes (:granule (:index-set index-set)))))
                                                   new-index-name (str index-set-id "_" idx-name)
                                                   new-index-names (update-in (get-concept-type-index-names context) [:index-names :granule] #(assoc % (keyword idx-name) new-index-name))
                                                   ;; This needs to set the in memory cache
-                                                  _ (set-concept-type-index-names context new-index-names)]
-                                              new-index-name)))
-                                        (get indexes :small_collections))
-         ]
+                                                  _ (set-concept-type-index-names context new-index-names)
+                                                  ;; this needs to set the search redis cache - this cache and the one above need to be combined into redis
 
+                                                  _ (set-search-index-names-cache context (keyword idx-name) new-index-name)]
+                                              new-index-name)))
+                                        (get indexes :small_collections))]
      (cond
        target-index-key
        [(get indexes target-index-key)]

indexer-app/src/cmr/indexer/services/index_set_service.clj

@@ -210,7 +210,10 @@
         es-store (indexer-util/context->es-store context)]
 
     (doseq [idx indices-w-config]
-      (es/update-index es-store idx))
+      (try
+        (es/update-index es-store idx)
+        (catch Exception e
+          (println "Caught exeption updating or creating" e))))
 
     (index-requested-index-set context index-set)))
 

indexer-app/src/cmr/indexer/system.clj

@@ -18,6 +18,7 @@
    [cmr.common.nrepl :as nrepl]
    [cmr.common.system :as common-sys]
    [cmr.elastic-utils.config :as es-config]
+   [cmr.elastic-utils.search.es-index-name-cache :as elastic-search-index-names-cache]
    [cmr.indexer.api.routes :as routes]
    [cmr.indexer.config :as config]
    [cmr.indexer.data.collection-granule-aggregation-cache :as cgac]
@@ -61,7 +62,9 @@
              :web (web/create-web-server (transmit-config/indexer-port) routes/make-api)
              :nrepl (nrepl/create-nrepl-if-configured (config/indexer-nrepl-port))
              :relative-root-url (transmit-config/indexer-relative-root-url)
-             :caches {af/acl-cache-key (af/create-consistent-acl-cache
+             :caches {;; first cache to set the search redis cache because the index set cache is different.
+                      elastic-search-index-names-cache/index-names-cache-key (elastic-search-index-names-cache/create-index-cache)
+                      af/acl-cache-key (af/create-consistent-acl-cache
                                         [:catalog-item :system-object :provider-object])
                       index-set/index-set-cache-key (consistent-cache/create-consistent-cache
                                                      {:hash-timeout-seconds (index-set-cache-consistent-timeout-seconds)

search-app/src/cmr/search/data/elastic_search_index.clj

@@ -42,6 +42,16 @@
     (apply dissoc granule-index-names (map keyword rebalancing-collections))))
 (comment 
   (println indexes)
+  (let [cache (hcache/context->cache context1 cache-key)]
+    (hcache/get-value cache cache-key :granule))
+  (:granule (:index-names (index-names-cache/refresh-index-names-cache context1)))
+  
+  ;      granule-index-names (or (hcache/get-value cache cache-key :granule)
+  ;                              (do
+  ;                                (index-names-cache/refresh-index-names-cache context)
+  ;                                (hcache/get-value cache cache-key :granule)))
+  ;      rebalancing-collections (hcache/get-value cache cache-key :rebalancing-collections)]
+  ;  (apply dissoc granule-index-names (map keyword rebalancing-collections)))
   )
 (defn- collection-concept-id->index-name
   "Return the granule index name for the input collection concept id"
@@ -57,10 +67,12 @@
   "Return the granule index names for the input collection concept ids"
   [context coll-concept-ids]
   (let [indexes (get-granule-index-names context)]
-    (distinct (map #(collection-concept-id->index-name indexes %) coll-concept-ids))))
+    (remove nil? 
+            (distinct (map #(collection-concept-id->index-name indexes %) coll-concept-ids)))))
 
 (comment
-  (println context)
+  (println context1)
+  (get-granule-index-names context1)
   )
 (defn- provider-ids->index-names
   "Return the granule index names for the input provider-ids"
@@ -97,9 +109,19 @@
       (format "%d_c*,%d_small_collections,-%d_collections*%s"
               index-set-id index-set-id index-set-id excluded-collections-str))))
 
+(comment
+  (println query1)
+  (get-granule-indexes context1 query1)
+  
+  (let [coll-concept-ids (seq (cex/extract-collection-concept-ids query1))]
+    (collection-concept-ids->index-names context1 coll-concept-ids))
+  )
+
 (defn- get-granule-indexes
   "Returns the granule indexes that should be searched based on the input query"
   [context query]
+  (def context1 context)
+  (def query1 query)
   (let [coll-concept-ids (seq (cex/extract-collection-concept-ids query))
         provider-ids (seq (pex/extract-provider-ids query))]
     (cond

system-int-test/src/cmr/system_int_test/system.clj

@@ -11,6 +11,7 @@
    [cmr.common.system :as common-sys]
    [cmr.common-app.services.kms-fetcher :as kf]
    [cmr.common-app.services.kms-lookup :as kl]
+   [cmr.elastic-utils.search.es-index-name-cache :as elastic-search-index-names-cache]
    [cmr.metadata-db.services.util :as mdb-util]
    [cmr.oracle.connection :as oracle]
    [cmr.system-int-test.utils.url-helper :as url]
@@ -42,7 +43,8 @@
    kl/kms-short-name-cache-key (kl/create-kms-short-name-cache)
    kl/kms-umm-c-cache-key (kl/create-kms-umm-c-cache)
    kl/kms-location-cache-key (kl/create-kms-location-cache)
-   kl/kms-measurement-cache-key (kl/create-kms-measurement-cache)})
+   kl/kms-measurement-cache-key (kl/create-kms-measurement-cache)
+   elastic-search-index-names-cache/index-names-cache-key (elastic-search-index-names-cache/create-index-cache)})
 
 (defn create-system
   "Returns a new instance of the whole application."