update rebalance api, comment out two tests, middle of discovering acl issue, fixed connection issue in int tests

Author: jmaeng72

access-control-app/src/cmr/access_control/api/routes.clj

@@ -311,14 +311,14 @@
             {ctx :request-context params :params}
             (acl/verify-ingest-management-permission ctx :update)
             (pv/validate-standard-params params)
-            (reindex-groups ctx))
+            (reindex-groups ctx)) ;; TODO CMR-10636 -- need to change this func?
 
       ;; Reindex all acls
       (POST "/reindex-acls"
             {ctx :request-context params :params}
             (acl/verify-ingest-management-permission ctx :update)
             (pv/validate-standard-params params)
-            (reindex-acls ctx))
+            (reindex-acls ctx)) ;; TODO CMR-10636 -- need to change this func?
 
       (if (access-control-config/enable-cmr-groups)
         (context "/groups" []

access-control-app/src/cmr/access_control/services/acl_service.clj

@@ -464,6 +464,7 @@
          (remove nil?)
          parse-single-string-multi-valued-bucket-lists)))
 
+;; TODO CMR-10636 - this is a duplicate method from elastic-utils and I think we can remove this and add it as a dependency in this project... but have to really test it out to see.
 (defmethod common-esi/concept-type->index-info :collection
   [_context _ query]
   ;; This function mirrors the multimethod definition in search.

elastic-utils-lib/src/cmr/elastic_utils/search/es_index.clj

@@ -21,10 +21,12 @@
 (defmulti concept-type->index-info
   "Returns index info based on input concept type. The map should contain a :type-name key along
    with an :index-name key. :index-name can refer to a single index or a comma separated string of
-   multiple index names."
+   multiple index names.
+   All index name strings in index-name are expected to be the same type as :type-name."
   (fn [_context concept-type _query]
     concept-type))
 
+;; TODO 10636 -- why are these defmethods here and in search app and duplicated in access-control?...need to consolidate this
 (defmethod concept-type->index-info :collection
   [_context _ query]
   {:index-name (if (:all-revisions? query)
@@ -152,10 +154,12 @@
     (catch ExceptionInfo e
       (handle-es-exception e scroll-id))))
 
+;; TODO 10636 FIX THIS
 ;; TODO 10636 this is hardcoded to index name...could it be better? Will these rules always be true?
 ;; TODO unit test this --  need a sys test as well, so that if any index is created or found, we will auto warn that something could break with this
 (defn get-es-cluster-name-from-index-name
   [index-name]
+  ;; NOTE: expecting index-name to represent only one index-name as a string
   ;(println "10636- INSIDE get-es-cluster-from-index-name. Given index-name = " index-name)
   (if
     (and (not (= index-name "collection_search_alias"))
@@ -165,24 +169,29 @@
              (= index-name "1_deleted_granules")
              (= index-name (str cmr.elastic-utils.config/gran-elastic-name "-index-sets")))))
     cmr.elastic-utils.config/gran-elastic-name
-    cmr.elastic-utils.config/non-gran-elastic-name)
-  )
+    cmr.elastic-utils.config/non-gran-elastic-name))
+
+(defn get-es-cluster-name-by-index-info-type
+  [index-info]
+  (if (= (:type index-info) "granule")
+    cmr.elastic-utils.config/gran-elastic-name
+    cmr.elastic-utils.config/non-gran-elastic-name))
 
 (defn- do-send-with-retry
   "Sends a query to ES, either normal or using a scroll query."
   [context index-info query max-retries]
-  ;; example index-info is {:index-name collection_search_alias, :type-name collection}
+  ;; example index-info is {:index-name collection_search_alias, :type-name collection} OR {:index-name 1_c*,1_small_collections,-1_collections*, :type-name granule}
   ;; will index-info always be one element or an array?
   ;(println "INSIDE do-send-with-retry with index info = " index-info " and query = " query)
   ;; index info =  {:index-name , :type-name granule}
   ;; query =  {:search_type query_then_fetch, :size 10, :from 0, :timeout 170s, :version true, :query {:bool {:must {:match_all {}}, :filter {:bool {:must ({:term {:collection-concept-id-doc-values C1200000001-JM_PROV1}} {:term {:concept-id G1200000002-JM_PROV1}})}}}}, :_source (:concept-id :revision-id :native-id-stored :provider-id-doc-values :metadata-format :revision-date-stored-doc-values :collection-concept-id-doc-values), :sort ({:provider-id-lowercase-doc-values {:order :asc}} {:start-date-doc-values {:order :asc}} {:concept-seq-id-long {:order asc}})}
-  (info "10636- es cluster we are using = " (get-es-cluster-name-from-index-name (:index-name index-info)))
+  (println "10636- INSIDE do-send-with-retry with index-info = " index-info ". Determined the es cluster is = " (get-es-cluster-name-by-index-info-type index-info))
   (try
     (if (pos? max-retries)
       (if-let [scroll-id (:scroll-id query)]
         (scroll-search context scroll-id)
         (es-helper/search
-          (context->conn context (get-es-cluster-name-from-index-name (:index-name index-info)))
+          (context->conn context (get-es-cluster-name-by-index-info-type index-info))
           (:index-name index-info)
           [(:type-name index-info)]
           query))

elastic-utils-lib/src/cmr/elastic_utils/search/query_execution.clj

@@ -9,6 +9,7 @@
 
 ;; *************************************************************************************************
 
+;; TODO CMR-10636 -- whereever this is called will have to change, we cannot just add the acls to the query anymore
 (defmulti add-acl-conditions-to-query
   "Adds conditions to the query to enforce ACLs."
   (fn [_context query]

indexer-app/src/cmr/indexer/api/routes.clj

@@ -41,7 +41,7 @@
       ;  ;;TODO 10636 fix this, we need to incorporate the non-gran-index-set resp too
       ;  (r/created gran-index-set-resp))
       ;; In the meantime, we are not going to allow this functionality.
-      {:status 200})
+      {:status 500})
 
     ;; respond with index-sets in elastic
     (GET "/" {request-context :request-context}

indexer-app/src/cmr/indexer/data/elasticsearch.clj

@@ -219,9 +219,7 @@
   [context index]
   (info (format "Deleting granule index %s from elastic" index))
   (try
-    (let [es-cluster-name (cmr.elastic-utils.search.es-index/get-es-cluster-name-from-index-name index)]
-      (esi/delete-index (indexer-util/context->conn context es-cluster-name) index)
-      )
+    (esi/delete-index (indexer-util/context->conn context cmr.elastic-utils.config/gran-elastic-name) index)
     (catch Throwable e
       (error e (str "Failed to delete granule index: "
                     (pr-str index))))))
@@ -438,7 +436,7 @@
    (delete-document context es-indexes es-type concept-id revision-id elastic-version nil))
   ([context es-indexes _es-type concept-id revision-id elastic-version options]
    (doseq [es-index es-indexes]
-     ;; Cannot use elastisch for deletion as we require special headers on delete
+     ;; Cannot use elasticsearch for deletion as we require special headers on delete
      (let [es-cluster-name (cmr.elastic-utils.search.es-index/get-es-cluster-name-from-index-name es-index)
            {:keys [admin-token]} (context->es-config context)
            {:keys [uri http-opts]} (indexer-util/context->conn context es-cluster-name)

search-app/src/cmr/search/api/concepts_search.clj

@@ -114,6 +114,7 @@
   "Invokes query service to parse the JSON query, find results and return
   the response."
   [ctx path-w-extension params headers json-query]
+  (println "INSIDE find-concepts-by-json-query")
   (let [concept-type (concept-type-path-w-extension->concept-type path-w-extension)
         params (core-api/process-params concept-type params path-w-extension headers mt/xml)
         _ (when (= :stac (:result-format params))
@@ -208,6 +209,7 @@
   "Invokes query service to parse the parameters query, find results, and
   return the response"
   [ctx path-w-extension params headers body]
+  (println "INSIDE find-concepts-by-parameters")
   (let [concept-type (concept-type-path-w-extension->concept-type path-w-extension)
         short-scroll-id (get headers (string/lower-case common-routes/SCROLL_ID_HEADER))
         scroll-id-and-search-params (core-api/get-scroll-id-and-search-params-from-cache ctx short-scroll-id)
@@ -244,6 +246,7 @@
     in the way that we need, we have to make two queries here to support CMR
     Harvesting. This can later be generalized easily, should the need arise."
   [ctx path-w-extension params headers body]
+  (println "INSIDE search find-concepts.")
   (let [content-type-header (get headers (string/lower-case common-routes/CONTENT_TYPE_HEADER))
         search-after (get headers (string/lower-case common-routes/SEARCH_AFTER_HEADER))
         _ (validate-search-after-value search-after)

system-int-test/src/cmr/system_int_test/utils/ingest_util.clj

@@ -393,7 +393,8 @@
                  :headers headers
                  :throw-exceptions false
                  :connection-manager (sys/conn-mgr)}
-         params (merge params (when accept-format {:accept accept-format}))]
+         params (merge params (when accept-format {:accept accept-format}))
+         _ (println "ingest params = " params)]
      (parse-ingest-response (client/request params) options))))
 
 (defn publish-draft

system-int-test/src/cmr/system_int_test/utils/search_util.clj

@@ -582,13 +582,15 @@
     (facets/parse-echo-facets-xml parsed)))
 
 (defn- parse-refs-response
-  "Parse the find-refs response based on expected format and retruns the parsed result"
+  "Parse the find-refs response based on expected format and returns the parsed result"
   [concept-type params options]
+  (println "INSIDE parse-refs-response")
   (let [;; params is not a map for catalog-rest additional attribute style tests,
         ;; we cannot destructing params as a map for the next two lines.
         echo-compatible (:echo-compatible params)
         include-facets (:include-facets params)
-        response (find-concepts-in-format mime-types/xml concept-type params options)]
+        response (find-concepts-in-format mime-types/xml concept-type params options)
+        _ (println "search response = " response)]
     (if (and echo-compatible include-facets)
       (parse-echo-facets-response response)
       (parse-reference-response echo-compatible response))))

system-int-test/test/cmr/system_int_test/search/acls/collection_test.clj

@@ -388,55 +388,84 @@
 ;; reindexed when ingest detects the acl hash has change.
 (deftest acl-change-test
   (let [coll1 (d/ingest "PROV1" (dc/collection-dif10 {:entry-title "coll1"}) {:format :dif10 :validate-keywords false})
-        coll2-umm (dc/collection {:entry-title "coll2" :short-name "short1"})
-        coll2-1 (d/ingest "PROV1" coll2-umm {:validate-keywords false})
-        ;; 2 versions of collection 2 will allow us to test the force reindex option after we
-        ;; force delete the latest version of coll2-2
-        coll2-2 (d/ingest "PROV1" (assoc-in coll2-umm [:product :short-name] "short2") {:validate-keywords false})
+        ;_ (println "coll1 = " coll1)
+        ;coll2-umm (dc/collection {:entry-title "coll2" :short-name "short1"})
+        ;coll2-1 (d/ingest "PROV1" coll2-umm {:validate-keywords false})
+        ;;; 2 versions of collection 2 will allow us to test the force reindex option after we
+        ;;; force delete the latest version of coll2-2
+        ;coll2-2 (d/ingest "PROV1" (assoc-in coll2-umm [:product :short-name] "short2") {:validate-keywords false})
         coll3 (d/ingest "PROV2" (dc/collection-dif10 {:entry-title "coll3"}) {:format :dif10 :validate-keywords false})
-        coll4 (d/ingest "PROV2" (dc/collection {:entry-title "coll4"}) {:validate-keywords false})
-
-        _ (index/wait-until-indexed)
+        ;_ (println "coll3 = " coll3)
+        ;coll4 (d/ingest "PROV2" (dc/collection {:entry-title "coll4"}) {:validate-keywords false})
+        ;
+        ;_ (index/wait-until-indexed)
         acl1 (e/grant-guest (s/context) (e/coll-catalog-item-id "PROV1" (e/coll-id ["coll1"])))
-        acl2 (e/grant-guest (s/context) (e/coll-catalog-item-id "PROV2" (e/coll-id ["coll3"])))]
+        acl2 (e/grant-guest (s/context) (e/coll-catalog-item-id "PROV2" (e/coll-id ["coll3"])))
+        ]
+
+    (testing "normal reindex collection permitted groups"
+      (println "TEST start")
+      ;(ingest/reindex-collection-permitted-groups (tc/echo-system-token))
+      ;(index/wait-until-indexed)
+      ;
+      ;(println "TEST finished reindex-collection-permitted-groups")
+      ;
+      ;;; before acls change
+      (d/assert-refs-match [coll1 coll3] (search/find-refs :collection {}))
+      ;(println "TEST finished first refs-match")
+
+      ;; EXPECTED
+      ;; {{:id "C1200000013-PROV1",
+      ;     :location "http://localhost:3003/concepts/C1200000013-PROV1/1",
+      ;     :name "coll1",
+      ;     :revision-id 1}
+      ;    {:id "C1200000015-PROV2",
+      ;     :location "http://localhost:3003/concepts/C1200000015-PROV2/1",
+      ;     :name "coll3",
+      ;     :revision-id 1}}
+
+      ;; Grant collection 2
+      ;(e/grant-guest (s/context) (e/coll-catalog-item-id "PROV1" (e/coll-id ["coll2"])))
+      ;(println "TEST finished grant guest")
+
+      ;; Ungrant collection 3
+      ;(e/ungrant (s/context) acl2)
+      ;(println "TEST finished ungrant")
+
+      ;; Try searching again before the reindexing
+      ;(d/assert-refs-match [coll1 coll3] (search/find-refs :collection {}))
+      ;(println "TEST finished refs match 2")
+
+      ;; Reindex collection permitted groups
+      ;(ingest/reindex-collection-permitted-groups (tc/echo-system-token))
+      ;(index/wait-until-indexed)
+      ;
+      ;(println "TEST finished reindex-collection-permitted-groups 2")
 
-    ;(testing "normal reindex collection permitted groups"
-    ;  (ingest/reindex-collection-permitted-groups (tc/echo-system-token))
-    ;  (index/wait-until-indexed)
-    ;
-    ;  ;; before acls change
-    ;  (d/assert-refs-match [coll1 coll3] (search/find-refs :collection {}))
+
+      ;; Search after reindexing
+      ;(d/assert-refs-match [coll1 coll2-2] (search/find-refs :collection {}))
+      ;(println "TEST finished assert-refs-match 3")
+
+      )
+
+    ;(testing "reindex all collections"
     ;
-    ;  ;; Grant collection 2
-    ;  (e/grant-guest (s/context) (e/coll-catalog-item-id "PROV1" (e/coll-id ["coll2"])))
-    ;  ;; Ungrant collection 3
-    ;  (e/ungrant (s/context) acl2)
+    ;  ;; Grant collection 4
+    ;  (e/grant-guest (s/context) (e/coll-catalog-item-id "PROV2" (e/coll-id ["coll4"])))
     ;
-    ;  ;; Try searching again before the reindexing
-    ;  (d/assert-refs-match [coll1 coll3] (search/find-refs :collection {}))
+    ;  ;; Try before reindexing
+    ;  (d/assert-refs-match [coll1 coll2-2] (search/find-refs :collection {}))
     ;
-    ;  ;; Reindex collection permitted groups
-    ;  (ingest/reindex-collection-permitted-groups (tc/echo-system-token))
+    ;  ;; Reindex all collections
+    ;  ;; Manually check the logs. It should say it's reindexing provider 1 and provider 3 as well.
+    ;  (ingest/reindex-all-collections)
     ;  (index/wait-until-indexed)
     ;
     ;  ;; Search after reindexing
-    ;  (d/assert-refs-match [coll1 coll2-2] (search/find-refs :collection {})))
-
-    (testing "reindex all collections"
-
-      ;; Grant collection 4
-      (e/grant-guest (s/context) (e/coll-catalog-item-id "PROV2" (e/coll-id ["coll4"])))
+    ;  (d/assert-refs-match [coll1 coll2-2 coll4] (search/find-refs :collection {})))
 
-      ;; Try before reindexing
-      (d/assert-refs-match [coll1 coll2-2] (search/find-refs :collection {}))
-
-      ;; Reindex all collections
-      ;; Manually check the logs. It should say it's reindexing provider 1 and provider 3 as well.
-      (ingest/reindex-all-collections)
-      (index/wait-until-indexed)
-
-      ;; Search after reindexing
-      (d/assert-refs-match [coll1 coll2-2 coll4] (search/find-refs :collection {})))))
+    ))
 
 ;; Verifies that tokens are cached by checking that a logged out token still works after it was
 ;; used. This isn't the desired behavior. It's just a side effect that shows it's working.