adding helper funcs

Author: jmaeng72

indexer-app/src/cmr/indexer/services/index_set_service.clj

@@ -181,6 +181,56 @@
   (when-let [error (index-cfg-validation index-set es-cluster-name)]
     (errors/throw-service-error :invalid-data error)))
 
+(defn get-index-set-revision
+  "Fetch a specific revision of an index-set from metadata-db.
+   If revision-id is nil, the latest revision is returned.
+   Returns a map with :index-set containing :revision-id and :deleted."
+  [context index-set-id revision-id]
+  (if-let [concept-id (meta-db/get-concept-id context :index-set "CMR" (str index-set-id))]
+    (let [concept (if revision-id
+                    (meta-db/get-concept context concept-id revision-id)
+                    (meta-db/get-latest-concept context concept-id))
+          metadata-map (when-let [metadata (:metadata concept)]
+                         (json/parse-string metadata true))]
+      (let [index-set (if (contains? metadata-map :index-set)
+                        (:index-set metadata-map)
+                        metadata-map)]
+        {:index-set (assoc index-set
+                           :revision-id (:revision-id concept)
+                           :deleted (true? (:deleted concept)))}))
+    (errors/throw-service-error :not-found
+                                (m/index-set-not-found-msg index-set-id))))
+
+(defn save-index-set-to-mdb
+  "Saves the provided combined-index-set to Metadata DB. Index set saved in DB is both the elastic and gran elastic index sets combined.
+   Returns the revision-id from Metadata DB."
+  [context combined-index-set]
+  (let [index-set-id (get-in combined-index-set [:index-set :id])
+        user-id (try
+                  (cxt/context->user-id context)
+                  (catch Exception _ "CMR"))]
+    (let [concept {:concept-type :index-set
+                   :native-id (str index-set-id)
+                   :provider-id "CMR"
+                   :metadata (json/generate-string combined-index-set)
+                   :user-id user-id
+                   :format "application/json"}]
+      (:revision-id (meta-db/save-concept context concept)))))
+
+(defn- save-combined-index-set-to-mdb
+  "Given a cluster's index-set (half of a combined index set), this func reconstructs the combined index-set of both elastic and gran elastic index-sets
+   and saves it to Metadata DB. Uses the provided index-set for the current cluster and fetches the other from ES.
+   Returns the revision-id from Metadata DB."
+  [context es-index-set es-cluster-name]
+  (let [index-set-id (get-in es-index-set [:index-set :id])
+        other-cluster (if (= es-cluster-name es-config/elastic-name)
+                        es-config/gran-elastic-name
+                        es-config/elastic-name)
+        ;; Use es/get-index-set directly to avoid :not-found exception during initial creation
+        other-index-set (es/get-index-set context other-cluster index-set-id)
+        combined-index-set (util/deep-merge other-index-set es-index-set)]
+    (save-index-set-to-mdb context combined-index-set)))
+
 (defn index-requested-index-set
   "Index requested index-set along with generated elastic index names"
   [context index-set es-cluster-name]
@@ -874,6 +924,21 @@
         (errors/throw-service-error :internal-error
                                     (format "Failed to rollback resharding for [%s]; see server logs." index))))))
 
+(defn sync-index-sets-from-db
+  "Fetches all latest non-deleted index-sets from Metadata DB and creates them in Elasticsearch."
+  [context]
+  (info "Syncing index-sets from Metadata DB to Elasticsearch...")
+  (let [index-sets (meta-db/find-concepts context {:latest true} :index-set)]
+    (doseq [concept index-sets]
+      (if (:deleted concept)
+        (info (format "Skipping deleted index-set [%s]" (:concept-id concept)))
+        (let [combined-index-set (json/parse-string (:metadata concept) true)
+              index-set-id (get-in combined-index-set [:index-set :id])]
+          (info (format "Restoring index-set [%s] (ID: %s) to Elasticsearch..."
+                        (:concept-id concept) index-set-id))
+          ;; Use put-index-set to ensure we update existing or create new
+          (put-index-set context combined-index-set))))))
+
 (defn reset
   "Put elastic in a clean state after deleting indices associated with index-sets and index-set docs."
   [context]

metadata-db-app/src/cmr/metadata_db/data/oracle/concepts/index_set.clj

@@ -0,0 +1,19 @@
+(ns cmr.metadata-db.data.oracle.concepts.index-set
+  "Implements multi-method variations for index-sets"
+  (:require [cmr.metadata-db.data.oracle.concepts :as c]))
+
+(defmethod c/db-result->concept-map :index-set
+  [concept-type db provider-id result]
+  (some-> (c/db-result->concept-map :default db provider-id result)
+          (assoc :concept-type :index-set)
+          (assoc :user-id (:user_id result))))
+
+;; Only "CMR" provider is supported now which is not considered a 'small' provider.
+;; If we ever associate real providers with index-sets then we will need to add support
+;; for small providers as well.
+(defmethod c/concept->insert-args [:index-set false]
+  [concept _]
+  (let [{user-id :user-id} concept
+        [cols values] (c/concept->common-insert-args concept)]
+    [(concat cols ["user_id"])
+     (concat values [user-id])]))

metadata-db-app/src/cmr/metadata_db/migrations/092_update_cmr_subscriptions_table.clj

@@ -7,12 +7,16 @@
   "Migrates the database up to version 92."
   []
   (println "cmr.metadata-db.migrations.092-update-cmr-sub-notifications-table up...")
-  (h/sql "ALTER TABLE METADATA_DB.CMR_SUB_NOTIFICATIONS DROP COLUMN AWS_ARN")
-  (h/sql "ALTER TABLE METADATA_DB.CMR_SUBSCRIPTIONS ADD AWS_ARN VARCHAR(2048) NULL"))
+  (when (h/column-exists? "CMR_SUB_NOTIFICATIONS" "AWS_ARN")
+    (h/sql "ALTER TABLE METADATA_DB.CMR_SUB_NOTIFICATIONS DROP COLUMN AWS_ARN"))
+  (when-not (h/column-exists? "CMR_SUBSCRIPTIONS" "AWS_ARN")
+    (h/sql "ALTER TABLE METADATA_DB.CMR_SUBSCRIPTIONS ADD AWS_ARN VARCHAR(2048) NULL")))
 
 (defn down
   "Migrates the database down from version 92."
   []
   (println "cmr.metadata-db.migrations.092-update-cmr-sub-notifications-table down.")
-  (h/sql "ALTER TABLE METADATA_DB.CMR_SUB_NOTIFICATIONS ADD AWS_ARN VARCHAR(2048) NULL")
-  (h/sql "ALTER TABLE METADATA_DB.CMR_SUBSCRIPTIONS DROP COLUMN AWS_ARN"))
+  (when-not (h/column-exists? "CMR_SUB_NOTIFICATIONS" "AWS_ARN")
+    (h/sql "ALTER TABLE METADATA_DB.CMR_SUB_NOTIFICATIONS ADD AWS_ARN VARCHAR(2048) NULL"))
+  (when (h/column-exists? "CMR_SUBSCRIPTIONS" "AWS_ARN")
+    (h/sql "ALTER TABLE METADATA_DB.CMR_SUBSCRIPTIONS DROP COLUMN AWS_ARN")))

metadata-db-app/src/cmr/metadata_db/services/concept_validations.clj

@@ -182,8 +182,15 @@
 (def humanizer-concept-validation
   "Builds a function that validates a concept map that has no provider and returns a list of errors"
   (v-util/compose-validations (conj base-concept-validations
-                                  concept-id-matches-concept-fields-validation-no-provider
-                                  humanizer-native-id-validation)))
+                                    concept-id-matches-concept-fields-validation-no-provider
+                                    humanizer-native-id-validation)))
+
+(def index-set-concept-validation
+  "Builds a function that validates a concept map that has no provider and returns a list of errors.
+   Index-sets are global system-level entities and are not associated with any specific data provider.
+   They are owned by the internal 'CMR' system provider."
+  (v-util/compose-validations (conj base-concept-validations
+                                    concept-id-matches-concept-fields-validation-no-provider)))
 
 (def validate-concept-default
   "Validates a concept. Throws an error if invalid."
@@ -205,6 +212,10 @@
   "validates a humanizer concept. Throws an error if invalid."
   (v-util/build-validator :invalid-data humanizer-concept-validation))
 
+(def validate-index-set-concept
+  "validates an index-set concept. Throws an error if invalid."
+  (v-util/build-validator :invalid-data index-set-concept-validation))
+
 (defmulti validate-concept
   "Validates a concept. Throws an error if invalid."
   (fn [concept]
@@ -226,6 +237,10 @@
   [concept]
   (validate-humanizer-concept concept))
 
+(defmethod validate-concept :index-set
+  [concept]
+  (validate-index-set-concept concept))
+
 (defmethod validate-concept :service-association
   [concept]
   (validate-association-concept concept))

metadata-db-app/src/cmr/metadata_db/services/messages.clj

@@ -212,6 +212,11 @@
   (format "Humanizer could not be associated with provider [%s]. Humanizer is system level entity."
           provider-id))
 
+(defn index-sets-only-system-level
+  [provider-id]
+  (format "Index-set could not be associated with provider [%s]. Index-set is system level entity."
+          provider-id))
+
 (defn variable-associations-only-system-level
   [provider-id]
   (format (str "Variable association could not be associated with provider [%s]. "

metadata-db-app/src/config/mdb_migrate_helper.clj

@@ -122,3 +122,13 @@
                  "select count(*) as c from all_sequences where sequence_name='CONCEPT_ID_SEQ'")
         seq-count (:c (first result))]
     (= 0 (int seq-count))))
+
+(defn column-exists?
+  "Returns true if the column exists in the table"
+  [table-name column-name]
+  (let [result (j/query
+                 (config/db)
+                 ["select count(*) as c from all_tab_columns where owner='METADATA_DB' and table_name=upper(?) and column_name=upper(?)"
+                  table-name column-name])
+        col-count (:c (first result))]
+    (not= 0 (int col-count))))

system-int-test/src/cmr/system_int_test/utils/index_util.clj

@@ -224,3 +224,29 @@
                :body (json/generate-string index-set)
                :connection-manager (s/conn-mgr)
                :throw-exceptions false}))
+
+(defn create-index-set
+  "Creates the given index-set in the indexer"
+  [index-set]
+  (client/post (url/indexer-index-sets-url)
+               {:headers {transmit-config/token-header (transmit-config/echo-system-token)
+                         "content-type" "application/json"}
+                :body (json/generate-string index-set)
+                :connection-manager (s/conn-mgr)
+                :throw-exceptions false}))
+
+(defn delete-index-set
+  "Deletes the index-set with the given id"
+  [id]
+  (client/delete (url/indexer-index-sets-by-id-url id)
+                 {:headers {transmit-config/token-header (transmit-config/echo-system-token)}
+                  :connection-manager (s/conn-mgr)
+                  :throw-exceptions false}))
+
+(defn sync-index-sets-from-db
+  "Triggers a sync of index-sets from database to elasticsearch"
+  []
+  (client/post (url/index-set-sync-url)
+               {:headers {transmit-config/token-header (transmit-config/echo-system-token)}
+                :connection-manager (s/conn-mgr)
+                :throw-exceptions false}))

system-int-test/src/cmr/system_int_test/utils/metadata_db_util.clj

@@ -99,3 +99,13 @@
                    :url (url/mdb-expired-concept-cleanup-url)
                    :headers {transmit-config/token-header (transmit-config/echo-system-token)}
                    :connection-manager (s/conn-mgr)}))
+
+(defn get-concept-id
+  "Returns a concept-id for the given concept-type, provider-id, and native-id"
+  [concept-type provider-id native-id]
+  (let [response (client/get (url/mdb-concept-id-url concept-type provider-id native-id)
+                             {:accept :json
+                              :throw-exceptions false
+                              :connection-manager (s/conn-mgr)})
+        body (json/decode (:body response) true)]
+    (get body :concept-id)))

system-int-test/src/cmr/system_int_test/utils/url_helper.clj

@@ -99,6 +99,15 @@
   []
   (format "http://localhost:%s/concepts" (transmit-config/metadata-db-port)))
 
+(defn mdb-concept-id-url
+  "URL to concept id in mdb."
+  [concept-type provider-id native-id]
+  (format "http://localhost:%s/concept-id/%s/%s/%s"
+          (transmit-config/metadata-db-port)
+          (name concept-type)
+          provider-id
+          native-id))
+
 (defn mdb-force-delete-concept-url
   [concept-id revision-id]
   (format "http://localhost:%s/concepts/force-delete/%s/%s"
@@ -673,6 +682,16 @@
   []
   (format "http://localhost:%s/index-sets/reset" (transmit-config/indexer-port)))
 
+(defn index-set-sync-url
+  "Sync index-sets from database to elasticsearch."
+  []
+  (format "http://localhost:%s/index-sets/sync-with-database" (transmit-config/indexer-port)))
+
+(defn indexer-index-sets-url
+  "Gets the index sets URL"
+  []
+  (format "http://localhost:%s/index-sets" (transmit-config/indexer-port)))
+
 (defn indexer-update-indexes
   "Updates the indexes in the indexer to update mappings and settings"
   []
@@ -683,6 +702,14 @@
   [id]
   (format "http://localhost:%s/index-sets/%s" (transmit-config/indexer-port) (str id)))
 
+(defn indexer-rebalance-url
+  [id concept-id]
+  (format "%s/rebalancing-collections/%s" (indexer-index-sets-by-id-url id) concept-id))
+
+(defn indexer-reshard-url
+  [id index-name]
+  (format "%s/reshard/%s" (indexer-index-sets-by-id-url id) index-name))
+
 (defn full-refresh-collection-granule-aggregate-cache-url
   []
   (format "http://localhost:%s/jobs/trigger-full-collection-granule-aggregate-cache-refresh"