Skip to content

Commit 2019755

Browse files
committed
Merge branch 'CMR-10636' into CMR-10636-jn
2 parents 8918130 + 2a111a0 commit 2019755

8 files changed

Lines changed: 106 additions & 122 deletions

File tree

bootstrap-app/src/cmr/bootstrap/data/bulk_index.clj

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,7 @@
7777
concept-batches (db/find-concepts-in-batches db provider params (:db-batch-size system) start-index)
7878
num-granules (index/bulk-index {:system (helper/get-indexer system)}
7979
concept-batches
80+
cmr.elastic-utils.config/gran-elastic-name
8081
{:target-index-key target-index-key})]
8182
(info "Indexed" num-granules "granule(s) for provider" provider-id "collection" collection-id)
8283
(when completion-message
@@ -98,7 +99,10 @@
9899
params {:concept-type :granule
99100
:provider-id provider-id}
100101
concept-batches (db/find-concepts-in-batches db provider params (:db-batch-size system) start-index)
101-
num-granules (index/bulk-index {:system (helper/get-indexer system)} concept-batches {})]
102+
num-granules (index/bulk-index {:system (helper/get-indexer system)}
103+
concept-batches
104+
cmr.elastic-utils.config/gran-elastic-name
105+
{})]
102106
(info "Indexed" num-granules "granule(s) for provider" provider-id)
103107
num-granules))
104108

@@ -110,7 +114,10 @@
110114
params {:concept-type :collection
111115
:provider-id provider-id}
112116
concept-batches (db/find-concepts-in-batches db provider params (:db-batch-size system))
113-
num-collections (index/bulk-index {:system (helper/get-indexer system)} concept-batches {})]
117+
num-collections (index/bulk-index {:system (helper/get-indexer system)}
118+
concept-batches
119+
cmr.elastic-utils.config/non-gran-elastic-name
120+
{})]
114121
(info "Indexed" num-collections "collection(s) for provider" provider-id)
115122
num-collections))
116123

@@ -128,12 +135,13 @@
128135
gran-count
129136
provider-id)))
130137

138+
;; TODO CMR-10636 fix this. We need to separate out the concepts and go to each seperate cluster
131139
(defn- bulk-index-concept-batches
132140
"Bulk index the given concept batches in both regular index and all revisions index."
133141
[system concept-batches]
134142
(let [indexer-context {:system (helper/get-indexer system)}]
135-
(index/bulk-index indexer-context concept-batches {:all-revisions-index? true})
136-
(index/bulk-index indexer-context concept-batches {})))
143+
(index/bulk-index indexer-context concept-batches nil {:all-revisions-index? true})
144+
(index/bulk-index indexer-context concept-batches nil {})))
137145

138146
(defn- index-concepts-by-provider
139147
"Bulk index concepts for the given provider and concept-type."
@@ -246,12 +254,12 @@
246254
{:concept-type concept-type :concept-id batch}
247255
(:db-batch-size system))]
248256
concept-batch)
249-
total (index/bulk-index {:system (helper/get-indexer system)} concept-batches)]
257+
total (index/bulk-index {:system (helper/get-indexer system)} concept-batches nil)]
250258

251259
;; for concept types that have all revisions index, also index the all revisions index
252260
(when-not (#{:tag :granule} concept-type)
253261
(index/bulk-index
254-
{:system (helper/get-indexer system)} concept-batches {:all-revisions-index? true}))
262+
{:system (helper/get-indexer system)} concept-batches nil {:all-revisions-index? true}))
255263

256264
(info "Indexed " total " concepts.")
257265
total))
@@ -281,7 +289,7 @@
281289
{:concept-type concept-type :concept-id batch}
282290
(:db-batch-size system))]
283291
(map #(assoc % :deleted true) concept-batch))
284-
total (index/bulk-index {:system (helper/get-indexer system)} concept-batches)]
292+
total (index/bulk-index {:system (helper/get-indexer system)} concept-batches nil)]
285293
(info "Deleted " total " concepts")
286294
total))
287295

elastic-utils-lib/src/cmr/elastic_utils/search/es_index.clj

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -179,9 +179,9 @@
179179
gran-cluster
180180
non-gran-cluster)))
181181

182-
(defn get-es-cluster-name-by-index-info-type
182+
(defn get-es-cluster-name-by-index-info-type-name
183183
[index-info]
184-
(if (= (:type index-info) "granule")
184+
(if (= (:type-name index-info) "granule")
185185
cmr.elastic-utils.config/gran-elastic-name
186186
cmr.elastic-utils.config/non-gran-elastic-name))
187187

@@ -193,16 +193,16 @@
193193
;(println "INSIDE do-send-with-retry with index info = " index-info " and query = " query)
194194
;; index info = {:index-name , :type-name granule}
195195
;; query = {:search_type query_then_fetch, :size 10, :from 0, :timeout 170s, :version true, :query {:bool {:must {:match_all {}}, :filter {:bool {:must ({:term {:collection-concept-id-doc-values C1200000001-JM_PROV1}} {:term {:concept-id G1200000002-JM_PROV1}})}}}}, :_source (:concept-id :revision-id :native-id-stored :provider-id-doc-values :metadata-format :revision-date-stored-doc-values :collection-concept-id-doc-values), :sort ({:provider-id-lowercase-doc-values {:order :asc}} {:start-date-doc-values {:order :asc}} {:concept-seq-id-long {:order asc}})}
196-
(println "10636- INSIDE do-send-with-retry with index-info = " index-info ". Determined the es cluster is = " (get-es-cluster-name-by-index-info-type index-info))
196+
(println "10636- INSIDE do-send-with-retry with index-info = " index-info ". Determined the es cluster is = " (get-es-cluster-name-by-index-info-type-name index-info))
197197
(try
198198
(if (pos? max-retries)
199199
(if-let [scroll-id (:scroll-id query)]
200200
(scroll-search context scroll-id)
201201
(es-helper/search
202-
(context->conn context (get-es-cluster-name-by-index-info-type index-info))
203-
(:index-name index-info)
204-
[(:type-name index-info)]
205-
query))
202+
(context->conn context (get-es-cluster-name-by-index-info-type-name index-info))
203+
(:index-name index-info)
204+
[(:type-name index-info)]
205+
query))
206206
(errors/throw-service-error :service-unavailable "Exhausted retries to execute ES query"))
207207

208208
(catch UnknownHostException _e

elastic-utils-lib/src/cmr/elastic_utils/search/query_execution.clj

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99

1010
;; *************************************************************************************************
1111

12-
;; TODO CMR-10636 -- whereever this is called will have to change, we cannot just add the acls to the query anymore
12+
;; TODO CMR-10636 -- where ever this is called will have to change, we cannot just add the acls to the query anymore
1313
(defmulti add-acl-conditions-to-query
1414
"Adds conditions to the query to enforce ACLs."
1515
(fn [_context query]
@@ -96,6 +96,7 @@
9696

9797
(defmethod execute-query :elasticsearch
9898
[context query]
99+
(println "INSIDE execute-query :elasticsearch")
99100
(let [[context processed-query] (concept-type-specific-query-processing
100101
context query)
101102
processed-query (pre-process-query-result-features context processed-query)

indexer-app/src/cmr/indexer/data/collection_granule_aggregation_cache.clj

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -229,7 +229,7 @@
229229
(meta-db/get-latest-concepts context)
230230
;; wrap it in a vector to make a batch to bulk index
231231
vector
232-
(index-service/bulk-index context)))))
232+
(index-service/bulk-index context cmr.elastic-utils.config/non-gran-elastic-name)))))
233233

234234
;; There's no existing value so a full refresh is required.
235235
(full-cache-refresh context))))

indexer-app/src/cmr/indexer/data/elasticsearch.clj

Lines changed: 15 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -360,31 +360,29 @@
360360
{:keys [_id status error]} resp-data]]
361361
(log/error (format "[%s] failed bulk indexing with status [%d] and error [%s]" _id status error)))))
362362

363-
;; TODO 10636- have to separate the docs in the bulk to non-gran and gran -- going to just disable this for now
363+
;; TODO 10636- check validity of this
364364
(defn bulk-index-autocomplete-suggestions
365365
"Save a batch of suggestion documents in Elasticsearch."
366366
[context docs]
367-
;(doseq [docs-batch (partition-all MAX_BULK_OPERATIONS_PER_REQUEST docs)]
368-
; (let [bulk-operations (cmr-bulk/create-bulk-index-operations docs-batch)
369-
; conn (indexer-util/context->conn context)
370-
; response (es-helper/bulk conn bulk-operations)]
371-
; (handle-bulk-index-response response)))
367+
(doseq [docs-batch (partition-all MAX_BULK_OPERATIONS_PER_REQUEST docs)]
368+
(let [bulk-operations (cmr-bulk/create-bulk-index-operations docs-batch)
369+
conn (indexer-util/context->conn context cmr.elastic-utils.config/non-gran-elastic-name)
370+
response (es-helper/bulk conn bulk-operations)]
371+
(handle-bulk-index-response response)))
372372
nil
373373
)
374374

375-
;; TODO 10636- have to separate the docs in the bulk to non-gran and gran -- going to just disable this for now
375+
;; TODO 10636- check validity of this
376376
(defn bulk-index-documents
377377
"Save a batch of documents in Elasticsearch."
378-
([context docs]
379-
(bulk-index-documents context docs nil))
380-
([context docs {:keys [all-revisions-index?]}]
381-
;(doseq [docs-batch (partition-all MAX_BULK_OPERATIONS_PER_REQUEST docs)]
382-
; (let [bulk-operations (cmr-bulk/create-bulk-index-operations docs-batch all-revisions-index?)
383-
; conn (indexer-util/context->conn context)
384-
; response (es-helper/bulk conn bulk-operations)]
385-
; (handle-bulk-index-response response)))
386-
nil
387-
))
378+
([context docs es-cluster-name]
379+
(bulk-index-documents context docs es-cluster-name nil))
380+
([context docs es-cluster-name {:keys [all-revisions-index?]}]
381+
(doseq [docs-batch (partition-all MAX_BULK_OPERATIONS_PER_REQUEST docs)]
382+
(let [bulk-operations (cmr-bulk/create-bulk-index-operations docs-batch all-revisions-index?)
383+
conn (indexer-util/context->conn context es-cluster-name)
384+
response (es-helper/bulk conn bulk-operations)]
385+
(handle-bulk-index-response response)))))
388386

389387
(defn get-es-cluster-conn
390388
[context es-index]

indexer-app/src/cmr/indexer/services/index_service.clj

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -139,12 +139,12 @@
139139
* :all-revisions-index? - true indicates this should be indexed into the all revisions index
140140
* :force-version? - true indicates that we should overwrite whatever is in elasticsearch with the
141141
latest regardless of whether the version in the database is older than the _version in elastic."
142-
([context concept-batches]
143-
(bulk-index context concept-batches nil))
144-
([context concept-batches options]
142+
([context concept-batches es-cluster-name]
143+
(bulk-index context concept-batches es-cluster-name nil))
144+
([context concept-batches es-cluster-name options]
145145
(reduce (fn [num-indexed batch]
146146
(let [batch (prepare-batch context batch options)]
147-
(es/bulk-index-documents context batch options)
147+
(es/bulk-index-documents context batch es-cluster-name options)
148148
(+ num-indexed (count batch))))
149149
0
150150
concept-batches)))
@@ -260,8 +260,11 @@
260260
:collection
261261
(determine-reindex-batch-size provider-id)
262262
{:provider-id provider-id :latest true})]
263-
(bulk-index context latest-collection-batches {:all-revisions-index? false
264-
:force-version? force-version?})))
263+
(println "latest-collection-batches = " latest-collection-batches)
264+
(bulk-index context
265+
latest-collection-batches
266+
cmr.elastic-utils.config/non-gran-elastic-name
267+
{:all-revisions-index? false :force-version? force-version?})))
265268

266269
(when (or (nil? all-revisions-index?) all-revisions-index?)
267270
;; Note that this will not unindex revisions that were removed directly from the database.
@@ -272,8 +275,10 @@
272275
:collection
273276
(determine-reindex-batch-size provider-id)
274277
{:provider-id provider-id})]
275-
(bulk-index context all-revisions-batches {:all-revisions-index? true
276-
:force-version? force-version?}))))))
278+
(bulk-index context
279+
all-revisions-batches
280+
cmr.elastic-utils.config/non-gran-elastic-name
281+
{:all-revisions-index? true :force-version? force-version?}))))))
277282

278283
(defconfig non-collection-reindex-batch-size
279284
"Batch size used for re-indexing other things besides collections."
@@ -289,7 +294,7 @@
289294
:tag
290295
(non-collection-reindex-batch-size)
291296
{:latest true})]
292-
(bulk-index context latest-tag-batches)))
297+
(bulk-index context latest-tag-batches cmr.elastic-utils.config/non-gran-elastic-name)))
293298

294299
(defn- time-to-visibility-text
295300
"This is the original log entry used by Splunk to report on time to index.

system-int-test/test/cmr/system_int_test/search/acls/collection_test.clj

Lines changed: 37 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -388,84 +388,56 @@
388388
;; reindexed when ingest detects the acl hash has change.
389389
(deftest acl-change-test
390390
(let [coll1 (d/ingest "PROV1" (dc/collection-dif10 {:entry-title "coll1"}) {:format :dif10 :validate-keywords false})
391-
;_ (println "coll1 = " coll1)
392-
;coll2-umm (dc/collection {:entry-title "coll2" :short-name "short1"})
393-
;coll2-1 (d/ingest "PROV1" coll2-umm {:validate-keywords false})
394-
;;; 2 versions of collection 2 will allow us to test the force reindex option after we
395-
;;; force delete the latest version of coll2-2
396-
;coll2-2 (d/ingest "PROV1" (assoc-in coll2-umm [:product :short-name] "short2") {:validate-keywords false})
391+
coll2-umm (dc/collection {:entry-title "coll2" :short-name "short1"})
392+
coll2-1 (d/ingest "PROV1" coll2-umm {:validate-keywords false})
393+
;; 2 versions of collection 2 will allow us to test the force reindex option after we
394+
;; force delete the latest version of coll2-2
395+
coll2-2 (d/ingest "PROV1" (assoc-in coll2-umm [:product :short-name] "short2") {:validate-keywords false})
397396
coll3 (d/ingest "PROV2" (dc/collection-dif10 {:entry-title "coll3"}) {:format :dif10 :validate-keywords false})
398-
;_ (println "coll3 = " coll3)
399-
;coll4 (d/ingest "PROV2" (dc/collection {:entry-title "coll4"}) {:validate-keywords false})
400-
;
401-
;_ (index/wait-until-indexed)
397+
coll4 (d/ingest "PROV2" (dc/collection {:entry-title "coll4"}) {:validate-keywords false})
398+
399+
_ (index/wait-until-indexed)
402400
acl1 (e/grant-guest (s/context) (e/coll-catalog-item-id "PROV1" (e/coll-id ["coll1"])))
403-
acl2 (e/grant-guest (s/context) (e/coll-catalog-item-id "PROV2" (e/coll-id ["coll3"])))
404-
]
401+
acl2 (e/grant-guest (s/context) (e/coll-catalog-item-id "PROV2" (e/coll-id ["coll3"])))]
405402

406403
(testing "normal reindex collection permitted groups"
407-
(println "TEST start")
408-
;(ingest/reindex-collection-permitted-groups (tc/echo-system-token))
409-
;(index/wait-until-indexed)
410-
;
411-
;(println "TEST finished reindex-collection-permitted-groups")
412-
;
413-
;;; before acls change
404+
(ingest/reindex-collection-permitted-groups (tc/echo-system-token))
405+
(index/wait-until-indexed)
406+
407+
;; before acls change
408+
(d/assert-refs-match [coll1 coll3] (search/find-refs :collection {}))
409+
410+
; Grant collection 2
411+
(e/grant-guest (s/context) (e/coll-catalog-item-id "PROV1" (e/coll-id ["coll2"])))
412+
413+
; Ungrant collection 3
414+
(e/ungrant (s/context) acl2)
415+
416+
; Try searching again before the reindexing
414417
(d/assert-refs-match [coll1 coll3] (search/find-refs :collection {}))
415-
;(println "TEST finished first refs-match")
416418

417-
;; EXPECTED
418-
;; {{:id "C1200000013-PROV1",
419-
; :location "http://localhost:3003/concepts/C1200000013-PROV1/1",
420-
; :name "coll1",
421-
; :revision-id 1}
422-
; {:id "C1200000015-PROV2",
423-
; :location "http://localhost:3003/concepts/C1200000015-PROV2/1",
424-
; :name "coll3",
425-
; :revision-id 1}}
419+
; Reindex collection permitted groups
420+
(ingest/reindex-collection-permitted-groups (tc/echo-system-token))
421+
(index/wait-until-indexed)
426422

427-
;; Grant collection 2
428-
;(e/grant-guest (s/context) (e/coll-catalog-item-id "PROV1" (e/coll-id ["coll2"])))
429-
;(println "TEST finished grant guest")
423+
; Search after reindexing
424+
(d/assert-refs-match [coll1 coll2-2] (search/find-refs :collection {})))
430425

431-
;; Ungrant collection 3
432-
;(e/ungrant (s/context) acl2)
433-
;(println "TEST finished ungrant")
426+
(testing "reindex all collections"
434427

435-
;; Try searching again before the reindexing
436-
;(d/assert-refs-match [coll1 coll3] (search/find-refs :collection {}))
437-
;(println "TEST finished refs match 2")
428+
;; Grant collection 4
429+
(e/grant-guest (s/context) (e/coll-catalog-item-id "PROV2" (e/coll-id ["coll4"])))
438430

439-
;; Reindex collection permitted groups
440-
;(ingest/reindex-collection-permitted-groups (tc/echo-system-token))
441-
;(index/wait-until-indexed)
442-
;
443-
;(println "TEST finished reindex-collection-permitted-groups 2")
431+
;; Try before reindexing
432+
(d/assert-refs-match [coll1 coll2-2] (search/find-refs :collection {}))
444433

434+
;; Reindex all collections
435+
;; Manually check the logs. It should say it's reindexing provider 1 and provider 3 as well.
436+
(ingest/reindex-all-collections)
437+
(index/wait-until-indexed)
445438

446439
;; Search after reindexing
447-
;(d/assert-refs-match [coll1 coll2-2] (search/find-refs :collection {}))
448-
;(println "TEST finished assert-refs-match 3")
449-
450-
)
451-
452-
;(testing "reindex all collections"
453-
;
454-
; ;; Grant collection 4
455-
; (e/grant-guest (s/context) (e/coll-catalog-item-id "PROV2" (e/coll-id ["coll4"])))
456-
;
457-
; ;; Try before reindexing
458-
; (d/assert-refs-match [coll1 coll2-2] (search/find-refs :collection {}))
459-
;
460-
; ;; Reindex all collections
461-
; ;; Manually check the logs. It should say it's reindexing provider 1 and provider 3 as well.
462-
; (ingest/reindex-all-collections)
463-
; (index/wait-until-indexed)
464-
;
465-
; ;; Search after reindexing
466-
; (d/assert-refs-match [coll1 coll2-2 coll4] (search/find-refs :collection {})))
467-
468-
))
440+
(d/assert-refs-match [coll1 coll2-2 coll4] (search/find-refs :collection {})))))
469441

470442
;; Verifies that tokens are cached by checking that a logged out token still works after it was
471443
;; used. This isn't the desired behavior. It's just a side effect that shows it's working.

0 commit comments

Comments
 (0)