Merge branch 'CMR-10636' into CMR-10636-jn

Author: indiejames

access-control-app/src/cmr/access_control/api/routes.clj

@@ -311,14 +311,14 @@
             {ctx :request-context params :params}
             (acl/verify-ingest-management-permission ctx :update)
             (pv/validate-standard-params params)
-            (reindex-groups ctx))
+            (reindex-groups ctx)) ;; TODO CMR-10636 -- need to change this func?
 
       ;; Reindex all acls
       (POST "/reindex-acls"
             {ctx :request-context params :params}
             (acl/verify-ingest-management-permission ctx :update)
             (pv/validate-standard-params params)
-            (reindex-acls ctx))
+            (reindex-acls ctx)) ;; TODO CMR-10636 -- need to change this func?
 
       (if (access-control-config/enable-cmr-groups)
         (context "/groups" []

access-control-app/src/cmr/access_control/services/acl_service.clj

@@ -464,6 +464,7 @@
          (remove nil?)
          parse-single-string-multi-valued-bucket-lists)))
 
+;; TODO CMR-10636 - this is a duplicate method from elastic-utils and I think we can remove this and add it as a dependency in this project... but have to really test it out to see.
 (defmethod common-esi/concept-type->index-info :collection
   [_context _ query]
   ;; This function mirrors the multimethod definition in search.

dev-system/src/cmr/dev_system/control.clj

@@ -119,7 +119,8 @@
               :when (get-in system [:apps service-name])]
         (reset-fn (app-context system service-name)))
       ;; After reset some elasticsearch indexes may not be initialized yet. We will check the status here
-      (elastic-conn/wait-for-healthy-elastic (get-in system [:apps :indexer :db]))
+      (elastic-conn/wait-for-healthy-elastic (get-in system [:apps :indexer :gran-elastic]))
+      (elastic-conn/wait-for-healthy-elastic (get-in system [:apps :indexer :non-gran-elastic]))
       (debug "dev system /reset complete")
       {:status 200})
 

elastic-utils-lib/src/cmr/elastic_utils/search/es_index.clj

@@ -22,10 +22,12 @@
 (defmulti concept-type->index-info
   "Returns index info based on input concept type. The map should contain a :type-name key along
    with an :index-name key. :index-name can refer to a single index or a comma separated string of
-   multiple index names."
+   multiple index names.
+   All index name strings in index-name are expected to be the same type as :type-name."
   (fn [_context concept-type _query]
     concept-type))
 
+;; TODO 10636 -- why are these defmethods here and in search app and duplicated in access-control?...need to consolidate this
 (defmethod concept-type->index-info :collection
   [_context _ query]
   {:index-name (if (:all-revisions? query)
@@ -61,9 +63,9 @@
    in a system using the :search-index key."
   [context es-cluster-name]
   (cond
-        (= es-cluster-name cmr.elastic-utils.config/non-gran-elastic-name) (get-in context [:system :non-gran-search-index])
-        (= es-cluster-name cmr.elastic-utils.config/gran-elastic-name) (get-in context [:system :gran-search-index])
-        :else (throw (Exception. (str "expected specific elastic name but got " es-cluster-name " instead.")))))
+    (= es-cluster-name cmr.elastic-utils.config/non-gran-elastic-name) (get-in context [:system :non-gran-search-index])
+    (= es-cluster-name cmr.elastic-utils.config/gran-elastic-name) (get-in context [:system :gran-search-index])
+    :else (throw (Exception. (str "expected specific elastic name but got " es-cluster-name " instead.")))))
 
 (defn context->conn
   "Returns the connection given a context. This assumes that the search index is always located in
@@ -153,11 +155,14 @@
     (catch ExceptionInfo e
       (handle-es-exception e scroll-id))))
 
+;; TODO 10636 FIX THIS
 ;; TODO 10636 this is hardcoded to index name...could it be better? Will these rules always be true?
 ;; TODO unit test this --  need a sys test as well, so that if any index is created or found, we will auto warn that something could break with this
 (defn get-es-cluster-name-from-index-name
   "Returns the Elasticsearch cluster name based on the index name."
   [index-name]
+  ;; NOTE: expecting index-name to represent only one index-name as a string
+  ;(println "10636- INSIDE get-es-cluster-from-index-name. Given index-name = " index-name)
   (let [gran-cluster cmr.elastic-utils.config/gran-elastic-name
         non-gran-cluster cmr.elastic-utils.config/non-gran-elastic-name
         gran-index-set-name (str gran-cluster "-index-sets")
@@ -170,29 +175,34 @@
                             (or (string/starts-with? index-name "1_c")
                                 (gran-specific-indices index-name)
                                 (= index-name gran-index-set-name)))]
-
     (if uses-gran-cluster?
       gran-cluster
       non-gran-cluster)))
 
+(defn get-es-cluster-name-by-index-info-type
+  [index-info]
+  (if (= (:type index-info) "granule")
+    cmr.elastic-utils.config/gran-elastic-name
+    cmr.elastic-utils.config/non-gran-elastic-name))
+
 (defn- do-send-with-retry
   "Sends a query to ES, either normal or using a scroll query."
   [context index-info query max-retries]
-  ;; example index-info is {:index-name collection_search_alias, :type-name collection}
+  ;; example index-info is {:index-name collection_search_alias, :type-name collection} OR {:index-name 1_c*,1_small_collections,-1_collections*, :type-name granule}
   ;; will index-info always be one element or an array?
   ;(println "INSIDE do-send-with-retry with index info = " index-info " and query = " query)
   ;; index info =  {:index-name , :type-name granule}
   ;; query =  {:search_type query_then_fetch, :size 10, :from 0, :timeout 170s, :version true, :query {:bool {:must {:match_all {}}, :filter {:bool {:must ({:term {:collection-concept-id-doc-values C1200000001-JM_PROV1}} {:term {:concept-id G1200000002-JM_PROV1}})}}}}, :_source (:concept-id :revision-id :native-id-stored :provider-id-doc-values :metadata-format :revision-date-stored-doc-values :collection-concept-id-doc-values), :sort ({:provider-id-lowercase-doc-values {:order :asc}} {:start-date-doc-values {:order :asc}} {:concept-seq-id-long {:order asc}})}
-  (info "10636- es cluster we are using = " (get-es-cluster-name-from-index-name (:index-name index-info)))
+  (println "10636- INSIDE do-send-with-retry with index-info = " index-info ". Determined the es cluster is = " (get-es-cluster-name-by-index-info-type index-info))
   (try
     (if (pos? max-retries)
       (if-let [scroll-id (:scroll-id query)]
         (scroll-search context scroll-id)
         (es-helper/search
-          (context->conn context (get-es-cluster-name-from-index-name (:index-name index-info)))
-          (:index-name index-info)
-          [(:type-name index-info)]
-          query))
+         (context->conn context (get-es-cluster-name-by-index-info-type index-info))
+         (:index-name index-info)
+         [(:type-name index-info)]
+         query))
       (errors/throw-service-error :service-unavailable "Exhausted retries to execute ES query"))
 
     (catch UnknownHostException _e
@@ -284,10 +294,10 @@
                       (set/rename-keys {:search-after :search_after})
                       util/remove-nil-keys)]
     (debug "Executing against indexes [" (:index-name index-info) "] the elastic query:"
-          (pr-str elastic-query)
-          "with sort" (pr-str sort-params)
-          "with aggregations" (pr-str aggregations)
-          "and highlights" (pr-str highlights))
+           (pr-str elastic-query)
+           "with sort" (pr-str sort-params)
+           "with aggregations" (pr-str aggregations)
+           "and highlights" (pr-str highlights))
     (when-let [scroll-id (:scroll-id query-map)]
       (debug "Using scroll-id" scroll-id))
     (when-let [search-after (:search_after query-map)]

elastic-utils-lib/src/cmr/elastic_utils/search/query_execution.clj

@@ -9,6 +9,7 @@
 
 ;; *************************************************************************************************
 
+;; TODO CMR-10636 -- whereever this is called will have to change, we cannot just add the acls to the query anymore
 (defmulti add-acl-conditions-to-query
   "Adds conditions to the query to enforce ACLs."
   (fn [_context query]

indexer-app/src/cmr/indexer/api/routes.clj

@@ -41,7 +41,7 @@
       ;  ;;TODO 10636 fix this, we need to incorporate the non-gran-index-set resp too
       ;  (r/created gran-index-set-resp))
       ;; In the meantime, we are not going to allow this functionality.
-      {:status 200})
+      {:status 500})
 
     ;; respond with index-sets in elastic
     (GET "/" {request-context :request-context}
@@ -77,26 +77,25 @@
         (index-set-svc/delete-index-set request-context id cmr.elastic-utils.config/non-gran-elastic-name)
         {:status 204})
 
-      ;; TODO 10636 We need to update all these endpoints to work with the new clusters
+      ;; TODO 10636 Updated. Need to test that it worked.
       (context "/rebalancing-collections/:concept-id" [concept-id]
 
         ;; Marks the collection as re-balancing in the index set.
         (POST "/start" {request-context :request-context params :params}
-          ;(acl/verify-ingest-management-permission request-context :update)
-          ;(index-set-svc/mark-collection-as-rebalancing request-context id concept-id (:target params))
-          ;{:status 200}
+          (acl/verify-ingest-management-permission request-context :update)
+          (index-set-svc/mark-collection-as-rebalancing request-context id concept-id (:target params))
           {:status 200})
 
         ;; Update the status of collection being rebalanced
         (POST "/update-status" {request-context :request-context params :params}
-          ;(acl/verify-ingest-management-permission request-context :update)
-          ;(index-set-svc/update-collection-rebalancing-status request-context id concept-id (:status params))
+          (acl/verify-ingest-management-permission request-context :update)
+          (index-set-svc/update-collection-rebalancing-status request-context id concept-id (:status params))
           {:status 200})
 
         ;; Marks the collection as completed rebalancing
         (POST "/finalize" {request-context :request-context}
-          ;(acl/verify-ingest-management-permission request-context :update)
-          ;(index-set-svc/finalize-collection-rebalancing request-context id concept-id)
+          (acl/verify-ingest-management-permission request-context :update)
+          (index-set-svc/finalize-collection-rebalancing request-context id concept-id)
           {:status 200})))))
 
 ;; Note for future. We should cleanup this API. It's not very well layed out.

indexer-app/src/cmr/indexer/data/elasticsearch.clj

@@ -219,9 +219,7 @@
   [context index]
   (info (format "Deleting granule index %s from elastic" index))
   (try
-    (let [es-cluster-name (cmr.elastic-utils.search.es-index/get-es-cluster-name-from-index-name index)]
-      (esi/delete-index (indexer-util/context->conn context es-cluster-name) index)
-      )
+    (esi/delete-index (indexer-util/context->conn context cmr.elastic-utils.config/gran-elastic-name) index)
     (catch Throwable e
       (error e (str "Failed to delete granule index: "
                     (pr-str index))))))
@@ -438,7 +436,7 @@
    (delete-document context es-indexes es-type concept-id revision-id elastic-version nil))
   ([context es-indexes _es-type concept-id revision-id elastic-version options]
    (doseq [es-index es-indexes]
-     ;; Cannot use elastisch for deletion as we require special headers on delete
+     ;; Cannot use elasticsearch for deletion as we require special headers on delete
      (let [es-cluster-name (cmr.elastic-utils.search.es-index/get-es-cluster-name-from-index-name es-index)
            {:keys [admin-token]} (context->es-config context)
            {:keys [uri http-opts]} (indexer-util/context->conn context es-cluster-name)

indexer-app/src/cmr/indexer/services/index_service.clj

@@ -888,7 +888,7 @@
 
 (defn reindex-all-collections
   "Reindexes all collections in all providers. This is only called in the indexer when humanizers
-  are updated and we only index the latest collection revision."
+  are updated, and we only index the latest collection revision."
   [context]
   (let [providers (map :provider-id (meta-db2/get-providers context))]
     (info "Sending events to reindex collections in all providers:" (pr-str providers))

search-app/src/cmr/search/api/concepts_search.clj

@@ -114,6 +114,7 @@
   "Invokes query service to parse the JSON query, find results and return
   the response."
   [ctx path-w-extension params headers json-query]
+  (println "INSIDE find-concepts-by-json-query")
   (let [concept-type (concept-type-path-w-extension->concept-type path-w-extension)
         params (core-api/process-params concept-type params path-w-extension headers mt/xml)
         _ (when (= :stac (:result-format params))
@@ -208,6 +209,7 @@
   "Invokes query service to parse the parameters query, find results, and
   return the response"
   [ctx path-w-extension params headers body]
+  (println "INSIDE find-concepts-by-parameters")
   (let [concept-type (concept-type-path-w-extension->concept-type path-w-extension)
         short-scroll-id (get headers (string/lower-case common-routes/SCROLL_ID_HEADER))
         scroll-id-and-search-params (core-api/get-scroll-id-and-search-params-from-cache ctx short-scroll-id)
@@ -244,6 +246,7 @@
     in the way that we need, we have to make two queries here to support CMR
     Harvesting. This can later be generalized easily, should the need arise."
   [ctx path-w-extension params headers body]
+  (println "INSIDE search find-concepts.")
   (let [content-type-header (get headers (string/lower-case common-routes/CONTENT_TYPE_HEADER))
         search-after (get headers (string/lower-case common-routes/SEARCH_AFTER_HEADER))
         _ (validate-search-after-value search-after)

system-int-test/src/cmr/system_int_test/utils/dev_system_util.clj

@@ -25,7 +25,7 @@
   (qb-side-api/wait-for-terminal-states)
   (try
     (client/post (url/dev-system-reset-url) (admin-connect-options))
-    (index/refresh-elastic-index)
+    (index/refresh-all-elastic-indexes)
     (load-kms-redis-cache) ;; This will cause the redis cache to reload
     (catch Exception e
       (error "Failed to send reset to dev-system\n" e)