update int tests

Author: jmaeng72

dev-system/src/cmr/dev_system/control.clj

@@ -119,7 +119,8 @@
               :when (get-in system [:apps service-name])]
         (reset-fn (app-context system service-name)))
       ;; After reset some elasticsearch indexes may not be initialized yet. We will check the status here
-      (elastic-conn/wait-for-healthy-elastic (get-in system [:apps :indexer :db]))
+      (elastic-conn/wait-for-healthy-elastic (get-in system [:apps :indexer :gran-elastic]))
+      (elastic-conn/wait-for-healthy-elastic (get-in system [:apps :indexer :non-gran-elastic]))
       (debug "dev system /reset complete")
       {:status 200})
 

indexer-app/src/cmr/indexer/api/routes.clj

@@ -77,26 +77,25 @@
         (index-set-svc/delete-index-set request-context id cmr.elastic-utils.config/non-gran-elastic-name)
         {:status 204})
 
-      ;; TODO 10636 We need to update all these endpoints to work with the new clusters
+      ;; TODO 10636 Updated. Need to test that it worked.
       (context "/rebalancing-collections/:concept-id" [concept-id]
 
         ;; Marks the collection as re-balancing in the index set.
         (POST "/start" {request-context :request-context params :params}
-          ;(acl/verify-ingest-management-permission request-context :update)
-          ;(index-set-svc/mark-collection-as-rebalancing request-context id concept-id (:target params))
-          ;{:status 200}
+          (acl/verify-ingest-management-permission request-context :update)
+          (index-set-svc/mark-collection-as-rebalancing request-context id concept-id (:target params))
           {:status 200})
 
         ;; Update the status of collection being rebalanced
         (POST "/update-status" {request-context :request-context params :params}
-          ;(acl/verify-ingest-management-permission request-context :update)
-          ;(index-set-svc/update-collection-rebalancing-status request-context id concept-id (:status params))
+          (acl/verify-ingest-management-permission request-context :update)
+          (index-set-svc/update-collection-rebalancing-status request-context id concept-id (:status params))
           {:status 200})
 
         ;; Marks the collection as completed rebalancing
         (POST "/finalize" {request-context :request-context}
-          ;(acl/verify-ingest-management-permission request-context :update)
-          ;(index-set-svc/finalize-collection-rebalancing request-context id concept-id)
+          (acl/verify-ingest-management-permission request-context :update)
+          (index-set-svc/finalize-collection-rebalancing request-context id concept-id)
           {:status 200})))))
 
 ;; Note for future. We should cleanup this API. It's not very well layed out.

indexer-app/src/cmr/indexer/services/index_service.clj

@@ -888,7 +888,7 @@
 
 (defn reindex-all-collections
   "Reindexes all collections in all providers. This is only called in the indexer when humanizers
-  are updated and we only index the latest collection revision."
+  are updated, and we only index the latest collection revision."
   [context]
   (let [providers (map :provider-id (meta-db2/get-providers context))]
     (info "Sending events to reindex collections in all providers:" (pr-str providers))

system-int-test/src/cmr/system_int_test/utils/dev_system_util.clj

@@ -25,7 +25,7 @@
   (qb-side-api/wait-for-terminal-states)
   (try
     (client/post (url/dev-system-reset-url) (admin-connect-options))
-    (index/refresh-elastic-index)
+    (index/refresh-all-elastic-indexes)
     (load-kms-redis-cache) ;; This will cause the redis cache to reload
     (catch Exception e
       (error "Failed to send reset to dev-system\n" e)

system-int-test/src/cmr/system_int_test/utils/index_util.clj

@@ -11,15 +11,16 @@
    [cmr.system-int-test.utils.url-helper :as url]
    [cmr.transmit.config :as transmit-config]))
 
-(defn refresh-elastic-index
+(defn refresh-all-elastic-indexes
   []
-  (client/post (url/elastic-refresh-url) {:connection-manager (s/conn-mgr)}))
+  (client/post (url/elastic-refresh-url cmr.elastic-utils.config/gran-elastic-name) {:connection-manager (s/conn-mgr)})
+  (client/post (url/elastic-refresh-url cmr.elastic-utils.config/non-gran-elastic-name) {:connection-manager (s/conn-mgr)}))
 
 (defn wait-until-indexed
   "Wait until ingested concepts have been indexed"
   []
   (qb-side-api/wait-for-terminal-states)
-  (refresh-elastic-index))
+  (refresh-all-elastic-indexes))
 
 (defn full-refresh-collection-granule-aggregate-cache
   "Triggers a full refresh of the collection granule aggregate cache in the indexer."
@@ -99,11 +100,12 @@
                      :query-params query-params})]
      response)))
 
+;; TODO 10636 - just a note that type-name is not used in this func...
 (defn doc-present?
   "If doc is present return true, otherwise return false"
-  [index-name type-name doc-id]
+  [index-name type-name doc-id elastic-name]
   (let [response (client/get
-                  (format "%s/%s/_doc/_search?q=_id:%s" (url/elastic-root) index-name doc-id)
+                  (format "%s/%s/_doc/_search?q=_id:%s" (url/elastic-root elastic-name) index-name doc-id)
                   {:throw-exceptions false
                    :connection-manager (s/conn-mgr)})
         body (json/decode (:body response) true)]
@@ -143,13 +145,13 @@
           (qb-side-api/set-message-queue-retry-behavior 0)
           (qb-side-api/set-message-queue-publish-timeout 10000))))))
 
-(defn delete-elasticsearch-index
-  "Helper to delete an elasticsearch index associated with a collection."
+(defn delete-gran-elastic-index
+  "Helper will delete the elasticsearch granule index associated with a collection."
   [coll]
   (let [index-name (string/replace (format "1_%s" (string/lower-case (:concept-id coll)))
                                    #"-" "_")]
     (warn "Deleting index " index-name)
-    (client/delete (format "%s/%s" (url/elastic-root) index-name)
+    (client/delete (format "%s/%s" (url/elastic-root cmr.elastic-utils.config/gran-elastic-name) index-name)
                    {:connection-manager (s/conn-mgr)})))
 
 (defn- query-for-granules-by-collection
@@ -162,19 +164,19 @@
       {:match_all {}}
       :filter {:term {:collection-concept-id-doc-values (:concept-id coll)}}}}}))
 
-(defn delete-granules-from-small-collections
+(defn delete-granules-from-small-collections-elastic-index
   "Helper to delete granules from the small collections index for the given collection."
   [coll]
-  (client/post (format "%s/1_small_collections/_delete_by_query" (url/elastic-root))
+  (client/post (format "%s/1_small_collections/_delete_by_query" (url/elastic-root cmr.elastic-utils.config/gran-elastic-name))
                  {:connection-manager (s/conn-mgr)
                   :body (query-for-granules-by-collection coll)
                   :content-type "application/json"}))
 
-(defn check-index-exists
-  "Helpper to check if elasticsearch index exists."
+(defn gran-elastic-index-exists?
+  "Helper to check if elasticsearch granule index exists."
   [coll]
   (let [index-name (string/replace (format "1_%s" (string/lower-case (:concept-id coll)))
                                    #"-" "_")]
-    (client/head (format "%s/%s" (url/elastic-root) index-name)
+    (client/head (format "%s/%s" (url/elastic-root cmr.elastic-utils.config/gran-elastic-name) index-name)
                  {:connection-manager (s/conn-mgr)
                   :throw-exceptions false})))

system-int-test/src/cmr/system_int_test/utils/url_helper.clj

@@ -56,16 +56,24 @@
 ;; Elasticsearch URLs
 
 (defn elastic-root
-  []
-  (format "http://localhost:%s" (es-config/elastic-port)))
+  [elastic-name]
+  (cond
+    (= elastic-name cmr.elastic-utils.config/gran-elastic-name)
+    (format "http://localhost:%s" (es-config/elastic-port))
+
+    (= elastic-name cmr.elastic-utils.config/non-gran-elastic-name)
+    (format "http://localhost:%s" (es-config/elastic-port-non-gran))
+
+    :else
+    (throw (Exception. (str "Given wrong elastic-name: " elastic-name " to create elastic root url.")))))
 
 (defn elastic-refresh-url
-  []
-  (str (elastic-root) "/_refresh"))
+  [elastic-name]
+  (str (elastic-root elastic-name) "/_refresh"))
 
 (defn elastic-delete-tag-url
   [id]
-  (format "%s/1_tags/_doc/%s" (elastic-root) id))
+  (format "%s/1_tags/_doc/%s" (elastic-root cmr.elastic-utils.config/non-gran-elastic-name) id))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; Metadata DB URLs

system-int-test/test/cmr/system_int_test/bootstrap/rebalance_collections_test.clj

@@ -435,7 +435,7 @@
      (search/clear-caches)
      ;; Delete the granules from small collections to show that search is still using the separate
      ;; index
-     (index/delete-granules-from-small-collections coll2)
+     (index/delete-granules-from-small-collections-elastic-index coll2)
      (index/wait-until-indexed)
      (assert-rebalance-status {:small-collections 0 :separate-index 2 :rebalancing-status "COMPLETE"} coll2)
      (is (= 2 (count (:refs (search/find-refs :granule {:concept-id (:concept-id coll1)})))))
@@ -462,8 +462,8 @@
      (is (= 4 (count (:refs (search/find-refs :granule {:concept-id (:concept-id coll1)})))))
      (is (= 2 (count (:refs (search/find-refs :granule {:concept-id (:concept-id coll2)})))))
      ;; Delete the collection specific indexes to show it has no impact on search after finalize
-     (index/delete-elasticsearch-index coll1)
-     (index/delete-elasticsearch-index coll2)
+     (index/delete-gran-elastic-index coll1)
+     (index/delete-gran-elastic-index coll2)
      (index/wait-until-indexed)
      (assert-rebalance-status {:small-collections 4 :rebalancing-status "NOT_REBALANCING"} coll1)
      (assert-rebalance-status {:small-collections 2 :rebalancing-status "NOT_REBALANCING"} coll2)

system-int-test/test/cmr/system_int_test/ingest/granule_bulk_update/granule_bulk_update_volume_test.clj

@@ -89,7 +89,7 @@
               (is (= "COMPLETE" (:task-status (ingest/granule-bulk-update-task-status task-id)))))
 
             (testing "The data is reflected in the updated values in search"
-              (index/refresh-elastic-index)
+              (index/refresh-all-elastic-indexes)
               (let [next-granules (-> (search/find-concepts-umm-json :granule {:granule-ur urs})
                                       :body
                                       (json/parse-string true)

system-int-test/test/cmr/system_int_test/ingest/misc/deleted_granule_index_test.clj

@@ -21,10 +21,10 @@
       (index/wait-until-indexed)
       (bootstrap/finalize-rebalance-collection (:concept-id collection))
       (index/wait-until-indexed)
-      (let [index-exists-before-delete-response (index/check-index-exists collection)
+      (let [index-exists-before-delete-response (index/gran-elastic-index-exists? collection)
             _ (ingest/delete-concept (data-core/umm-c-collection->concept collection :echo10) {})
             _ (index/wait-until-indexed)
-            index-exists-after-delete-response (index/check-index-exists collection)]
+            index-exists-after-delete-response (index/gran-elastic-index-exists? collection)]
         (is (= 200 (:status index-exists-before-delete-response)))
         (is (= 404 (:status index-exists-after-delete-response))))))
   (testing "Ingest collection, rebalance collection, delete collection, ingest collection, ingest granule, check index exists again"
@@ -37,7 +37,7 @@
       (index/wait-until-indexed)
       (ingest/delete-concept (data-core/umm-c-collection->concept collection :echo10) {})
       (index/wait-until-indexed)
-      (let [index-not-exists-before-reingest-response (index/check-index-exists collection)
+      (let [index-not-exists-before-reingest-response (index/gran-elastic-index-exists? collection)
             new-collection (data-core/ingest-umm-spec-collection "PROV1"
                                                                  (data-umm-c/collection {})
                                                                  {:validate-keywords false})
@@ -46,6 +46,6 @@
                                                    dissoc :ShortName :Version))
             _ (index/wait-until-indexed)
 
-            index-not-exists-after-reingest-response (index/check-index-exists collection)]
+            index-not-exists-after-reingest-response (index/gran-elastic-index-exists? collection)]
         (is (= 404 (:status index-not-exists-before-reingest-response)))
         (is (= 200 (:status index-not-exists-after-reingest-response)))))))

system-int-test/test/cmr/system_int_test/ingest/misc/deleted_granules_index_test.clj

@@ -25,7 +25,8 @@
   [concept-id]
   (index/doc-present? deleted-granule/deleted-granule-index-name
                       deleted-granule/deleted-granule-type-name
-                      concept-id))
+                      concept-id
+                      cmr.elastic-utils.config/gran-elastic-name))
 
 (defn- find-deleted-granules
   "Calls get-deleted-granules endpoint and returns parsed items from response"