fixed various search and ingest tests

Author: jmaeng72

access-control-app/src/cmr/access_control/api/routes.clj

@@ -117,7 +117,6 @@
 (defn- create-acl
   "Returns a Ring response with the result of trying to create the ACL with the given request body."
   [request-ctx headers body]
-  (println "INSIDE create-acl")
   (validate-content-type headers)
   (acl-schema/validate-acl-json body)
   (try

access-control-app/src/cmr/access_control/services/acl_util.clj

@@ -74,7 +74,6 @@
   (let [acl-concept (merge (acl->base-concept context acl)
                            {:revision-id 1
                             :native-id (str (java.util.UUID/randomUUID))})
-        _ (println "acl-concept = " acl-concept)
         resp (mdb/save-concept context acl-concept)]
     ;; index the saved ACL here to make ingest synchronous
     (index/index-acl context

bootstrap-app/src/cmr/bootstrap/data/bulk_index.clj

@@ -77,6 +77,7 @@
         concept-batches (db/find-concepts-in-batches db provider params (:db-batch-size system) start-index)
         num-granules (index/bulk-index {:system (helper/get-indexer system)}
                                        concept-batches
+                                       cmr.elastic-utils.config/gran-elastic-name
                                        {:target-index-key target-index-key})]
     (info "Indexed" num-granules "granule(s) for provider" provider-id "collection" collection-id)
     (when completion-message
@@ -98,7 +99,10 @@
         params {:concept-type :granule
                 :provider-id provider-id}
         concept-batches (db/find-concepts-in-batches db provider params (:db-batch-size system) start-index)
-        num-granules (index/bulk-index {:system (helper/get-indexer system)} concept-batches {})]
+        num-granules (index/bulk-index {:system (helper/get-indexer system)}
+                                       concept-batches
+                                       cmr.elastic-utils.config/gran-elastic-name
+                                       {})]
     (info "Indexed" num-granules "granule(s) for provider" provider-id)
     num-granules))
 
@@ -110,7 +114,10 @@
         params {:concept-type :collection
                 :provider-id provider-id}
         concept-batches (db/find-concepts-in-batches db provider params (:db-batch-size system))
-        num-collections (index/bulk-index {:system (helper/get-indexer system)} concept-batches {})]
+        num-collections (index/bulk-index {:system (helper/get-indexer system)}
+                                          concept-batches
+                                          cmr.elastic-utils.config/non-gran-elastic-name
+                                          {})]
     (info "Indexed" num-collections "collection(s) for provider" provider-id)
     num-collections))
 
@@ -128,12 +135,13 @@
             gran-count
             provider-id)))
 
+;; TODO CMR-10636 fix this. We need to separate out the concepts and go to each seperate cluster
 (defn- bulk-index-concept-batches
   "Bulk index the given concept batches in both regular index and all revisions index."
   [system concept-batches]
   (let [indexer-context {:system (helper/get-indexer system)}]
-    (index/bulk-index indexer-context concept-batches {:all-revisions-index? true})
-    (index/bulk-index indexer-context concept-batches {})))
+    (index/bulk-index indexer-context concept-batches nil {:all-revisions-index? true})
+    (index/bulk-index indexer-context concept-batches nil {})))
 
 (defn- index-concepts-by-provider
   "Bulk index concepts for the given provider and concept-type."
@@ -246,12 +254,12 @@
                                                                          {:concept-type concept-type :concept-id batch}
                                                                          (:db-batch-size system))]
                           concept-batch)
-        total (index/bulk-index {:system (helper/get-indexer system)} concept-batches)]
+        total (index/bulk-index {:system (helper/get-indexer system)} concept-batches nil)]
 
     ;; for concept types that have all revisions index, also index the all revisions index
     (when-not (#{:tag :granule} concept-type)
       (index/bulk-index
-       {:system (helper/get-indexer system)} concept-batches {:all-revisions-index? true}))
+       {:system (helper/get-indexer system)} concept-batches nil {:all-revisions-index? true}))
 
     (info "Indexed " total " concepts.")
     total))
@@ -281,7 +289,7 @@
                                                                          {:concept-type concept-type :concept-id batch}
                                                                          (:db-batch-size system))]
                           (map #(assoc % :deleted true) concept-batch))
-        total (index/bulk-index {:system (helper/get-indexer system)} concept-batches)]
+        total (index/bulk-index {:system (helper/get-indexer system)} concept-batches nil)]
     (info "Deleted " total " concepts")
     total))
 

common-app-lib/src/cmr/common_app/services/search.clj

@@ -104,13 +104,12 @@
         [result-gen-time result-str] (u/time-execution
                                       (search-results->response
                                        context query (assoc results :took query-execution-time)))]
-    (println "query-execution-time:" query-execution-time "result-gen-time:" result-gen-time)
+    (info "query-execution-time:" query-execution-time "result-gen-time:" result-gen-time)
     [results result-str]))
 
 (defn find-concepts
   "Executes a search for concepts using the given query."
   [context _concept-type query]
-  (println "INSIDE find-concepts in search")
   (validate-query context query)
   ;; If the scroll-id is not nil, first look in the cache to see if there is a deferred result and
   ;; use that if so. If the scroll-id is not set and scroll is set to 'defer' then store the

elastic-utils-lib/src/cmr/elastic_utils/es_helper.clj

@@ -18,7 +18,6 @@
         qp (merge {:track_total_hits true}
                   (select-keys opts qk))
         body (apply dissoc (concat [opts] qk))]
-    (println "INSIDE final search call to ES with url = " (rest/search-url conn (join-names index)) " and body = " body " with query params = " qp)
     (rest/post conn (rest/search-url conn (join-names index))
                {:content-type :json
                 :body body

elastic-utils-lib/src/cmr/elastic_utils/search/es_index.clj

@@ -171,9 +171,9 @@
     cmr.elastic-utils.config/gran-elastic-name
     cmr.elastic-utils.config/non-gran-elastic-name))
 
-(defn get-es-cluster-name-by-index-info-type
+(defn get-es-cluster-name-by-index-info-type-name
   [index-info]
-  (if (= (:type index-info) "granule")
+  (if (= (:type-name index-info) "granule")
     cmr.elastic-utils.config/gran-elastic-name
     cmr.elastic-utils.config/non-gran-elastic-name))
 
@@ -185,13 +185,13 @@
   ;(println "INSIDE do-send-with-retry with index info = " index-info " and query = " query)
   ;; index info =  {:index-name , :type-name granule}
   ;; query =  {:search_type query_then_fetch, :size 10, :from 0, :timeout 170s, :version true, :query {:bool {:must {:match_all {}}, :filter {:bool {:must ({:term {:collection-concept-id-doc-values C1200000001-JM_PROV1}} {:term {:concept-id G1200000002-JM_PROV1}})}}}}, :_source (:concept-id :revision-id :native-id-stored :provider-id-doc-values :metadata-format :revision-date-stored-doc-values :collection-concept-id-doc-values), :sort ({:provider-id-lowercase-doc-values {:order :asc}} {:start-date-doc-values {:order :asc}} {:concept-seq-id-long {:order asc}})}
-  (println "10636- INSIDE do-send-with-retry with index-info = " index-info ". Determined the es cluster is = " (get-es-cluster-name-by-index-info-type index-info))
+  (println "10636- INSIDE do-send-with-retry with index-info = " index-info ". Determined the es cluster is = " (get-es-cluster-name-by-index-info-type-name index-info))
   (try
     (if (pos? max-retries)
       (if-let [scroll-id (:scroll-id query)]
         (scroll-search context scroll-id)
         (es-helper/search
-          (context->conn context (get-es-cluster-name-by-index-info-type index-info))
+          (context->conn context (get-es-cluster-name-by-index-info-type-name index-info))
           (:index-name index-info)
           [(:type-name index-info)]
           query))

elastic-utils-lib/src/cmr/elastic_utils/search/query_execution.clj

@@ -103,7 +103,7 @@
         elastic-results (->> processed-query
                              (#(if (or (tc/echo-system-token? context) (:skip-acls? %))
                                  %
-                                 (add-acl-conditions-to-query context %))) ;; TODO CMR-10636 - is this going to be an issue?
+                                 (add-acl-conditions-to-query context %)))
                              (c2s/reduce-query context)
                              (idx/execute-query context))
         query-results (rc/elastic-results->query-results context processed-query elastic-results)]

indexer-app/src/cmr/indexer/data/collection_granule_aggregation_cache.clj

@@ -229,7 +229,7 @@
                (meta-db/get-latest-concepts context)
                ;; wrap it in a vector to make a batch to bulk index
                vector
-               (index-service/bulk-index context)))))
+               (index-service/bulk-index context cmr.elastic-utils.config/non-gran-elastic-name)))))
 
       ;; There's no existing value so a full refresh is required.
       (full-cache-refresh context))))

indexer-app/src/cmr/indexer/data/elasticsearch.clj

@@ -360,31 +360,29 @@
                   {:keys [_id status error]} resp-data]]
       (log/error (format "[%s] failed bulk indexing with status [%d] and error [%s]" _id status error)))))
 
-;; TODO 10636- have to separate the docs in the bulk to non-gran and gran -- going to just disable this for now
+;; TODO 10636- check validity of this
 (defn bulk-index-autocomplete-suggestions
   "Save a batch of suggestion documents in Elasticsearch."
   [context docs]
-  ;(doseq [docs-batch (partition-all MAX_BULK_OPERATIONS_PER_REQUEST docs)]
-  ;  (let [bulk-operations (cmr-bulk/create-bulk-index-operations docs-batch)
-  ;        conn (indexer-util/context->conn context)
-  ;        response (es-helper/bulk conn bulk-operations)]
-  ;    (handle-bulk-index-response response)))
+  (doseq [docs-batch (partition-all MAX_BULK_OPERATIONS_PER_REQUEST docs)]
+    (let [bulk-operations (cmr-bulk/create-bulk-index-operations docs-batch)
+          conn (indexer-util/context->conn context cmr.elastic-utils.config/non-gran-elastic-name)
+          response (es-helper/bulk conn bulk-operations)]
+      (handle-bulk-index-response response)))
   nil
   )
 
-;; TODO 10636- have to separate the docs in the bulk to non-gran and gran -- going to just disable this for now
+;; TODO 10636- check validity of this
 (defn bulk-index-documents
   "Save a batch of documents in Elasticsearch."
-  ([context docs]
-   (bulk-index-documents context docs nil))
-  ([context docs {:keys [all-revisions-index?]}]
-   ;(doseq [docs-batch (partition-all MAX_BULK_OPERATIONS_PER_REQUEST docs)]
-   ;  (let [bulk-operations (cmr-bulk/create-bulk-index-operations docs-batch all-revisions-index?)
-   ;        conn (indexer-util/context->conn context)
-   ;        response (es-helper/bulk conn bulk-operations)]
-   ;    (handle-bulk-index-response response)))
-   nil
-   ))
+  ([context docs es-cluster-name]
+   (bulk-index-documents context docs es-cluster-name nil))
+  ([context docs es-cluster-name {:keys [all-revisions-index?]}]
+   (doseq [docs-batch (partition-all MAX_BULK_OPERATIONS_PER_REQUEST docs)]
+     (let [bulk-operations (cmr-bulk/create-bulk-index-operations docs-batch all-revisions-index?)
+           conn (indexer-util/context->conn context es-cluster-name)
+           response (es-helper/bulk conn bulk-operations)]
+       (handle-bulk-index-response response)))))
 
 (defn get-es-cluster-conn
   [context es-index]

indexer-app/src/cmr/indexer/services/index_service.clj

@@ -139,12 +139,12 @@
   * :all-revisions-index? - true indicates this should be indexed into the all revisions index
   * :force-version? - true indicates that we should overwrite whatever is in elasticsearch with the
   latest regardless of whether the version in the database is older than the _version in elastic."
-  ([context concept-batches]
-   (bulk-index context concept-batches nil))
-  ([context concept-batches options]
+  ([context concept-batches es-cluster-name]
+   (bulk-index context concept-batches es-cluster-name nil))
+  ([context concept-batches es-cluster-name options]
    (reduce (fn [num-indexed batch]
              (let [batch (prepare-batch context batch options)]
-               (es/bulk-index-documents context batch options)
+               (es/bulk-index-documents context batch es-cluster-name options)
                (+ num-indexed (count batch))))
            0
            concept-batches)))
@@ -260,8 +260,11 @@
                                         :collection
                                         (determine-reindex-batch-size provider-id)
                                         {:provider-id provider-id :latest true})]
-         (bulk-index context latest-collection-batches {:all-revisions-index? false
-                                                        :force-version? force-version?})))
+         (println "latest-collection-batches = " latest-collection-batches)
+         (bulk-index context
+                     latest-collection-batches
+                     cmr.elastic-utils.config/non-gran-elastic-name
+                     {:all-revisions-index? false :force-version? force-version?})))
 
      (when (or (nil? all-revisions-index?) all-revisions-index?)
        ;; Note that this will not unindex revisions that were removed directly from the database.
@@ -272,8 +275,10 @@
                                     :collection
                                     (determine-reindex-batch-size provider-id)
                                     {:provider-id provider-id})]
-         (bulk-index context all-revisions-batches {:all-revisions-index? true
-                                                    :force-version? force-version?}))))))
+         (bulk-index context
+                     all-revisions-batches
+                     cmr.elastic-utils.config/non-gran-elastic-name
+                     {:all-revisions-index? true :force-version? force-version?}))))))
 
 (defconfig non-collection-reindex-batch-size
   "Batch size used for re-indexing other things besides collections."
@@ -289,7 +294,7 @@
                             :tag
                             (non-collection-reindex-batch-size)
                             {:latest true})]
-    (bulk-index context latest-tag-batches)))
+    (bulk-index context latest-tag-batches cmr.elastic-utils.config/non-gran-elastic-name)))
 
 (defn- time-to-visibility-text
   "This is the original log entry used by Splunk to report on time to index.