Skip to content

Commit 40abbb7

Browse files
jmaeng72jmaeng72
committed
Merge branch 'master' of https://github.com/nasa/Common-Metadata-Repository into CMR-11398
2 parents d7e1282 + 29da8cc commit 40abbb7

22 files changed

Lines changed: 271 additions & 57 deletions

File tree

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ profiles.clj
2424
dev-system/local.edn
2525
*pycache*
2626
.portal
27+
.snyk
2728

2829
###############################
2930
### Test Files

‎access-control-app/project.clj‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,15 +37,19 @@
3737
[compojure "1.6.1"
3838
:exclusions [commons-fileupload]]
3939
[gov.nasa.earthdata/cmr-site-templates "0.1.1-SNAPSHOT"]
40+
;; replaces org.bouncycastle/bcpkix-jdk15on
41+
[org.bouncycastle/bcpkix-jdk18on "1.85"]
4042
[org.clojure/clojure "1.11.2"]
4143
[org.clojure/tools.reader "1.3.2"]
4244
[org.eclipse.jetty.ee9/jetty-ee9-servlet "12.1.10"]
4345
[org.eclipse.jetty/jetty-http "12.1.8"]
4446
[org.eclipse.jetty/jetty-util "12.1.8"]
4547
[ring/ring-core "1.15.4"]
48+
49+
[ring/ring-codec "1.3.0" :exclusions [org.bouncycastle/bcpkix-jdk15on]]
50+
4651
[ring/ring-jetty-adapter "1.15.4"
4752
:exclusions [org.eclipse.jetty.ee9/jetty-ee9-servlet]]
48-
[ring/ring-codec "1.3.0"]
4953
[ring/ring-json "0.5.1"]]
5054
project-dependencies)
5155
:plugins [[lein-modules "0.3.11"]
@@ -59,13 +63,15 @@
5963
:dependency-check {:output-format [:all]
6064
:suppression-file "resources/security/suppression.xml"}}
6165
:dev {:dependencies [[org.apache.httpcomponents/httpclient "4.5.13"]
66+
[org.bouncycastle/bcpkix-jdk18on "1.85"]
6267
[org.clojure/core.async "0.4.500"]
6368
[org.clojure/tools.namespace "0.2.11"]
6469
[org.clojure/tools.nrepl "0.2.13"]
6570
[org.eclipse.jetty.ee9/jetty-ee9-servlet "12.1.10"]
6671
[pjstadig/humane-test-output "0.9.0"]
6772
[proto-repl "0.3.1"]
68-
[ring/ring-codec "1.3.0"]
73+
[ring/ring-codec "1.3.0" :exclusions
74+
[org.bouncycastle/bcpkix-jdk15on]]
6975
[ring/ring-jetty-adapter "1.15.4"
7076
:exclusions [org.eclipse.jetty.ee9/jetty-ee9-servlet]]
7177
[ring-mock "0.1.5"]]

‎bootstrap-app/project.clj‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@
2121
[nasa-cmr/cmr-virtual-product-app "0.1.0-SNAPSHOT"]
2222
[org.apache.httpcomponents/httpclient "4.5.13"]
2323
[org.apache.httpcomponents/httpcore "4.4.10"]
24+
;; replaces org.bouncycastle/bcpkix-jdk15on
25+
[org.bouncycastle/bcpkix-jdk18on "1.85"]
2426
[org.clojure/clojure "1.11.2"]
2527
[org.clojure/tools.nrepl "0.2.13"]
2628
[org.clojure/tools.reader "1.3.2"]
@@ -29,9 +31,9 @@
2931
[org.eclipse.jetty/jetty-http "12.1.8"]
3032
[org.eclipse.jetty/jetty-util "12.1.8"]
3133
[ring/ring-core "1.15.4"]
34+
[ring/ring-codec "1.3.0" :exclusions [org.bouncycastle/bcpkix-jdk15on]]
3235
[ring/ring-jetty-adapter "1.15.4"
3336
:exclusions [org.eclipse.jetty.ee9/jetty-ee9-servlet]]
34-
[ring/ring-codec "1.3.0"]
3537
[ring/ring-json "0.5.1"]]
3638
:plugins [[io.github.jaybarra/drift "1.5.4.2-SNAPSHOT" :exclusions [clojure-tools]]
3739
[lein-exec "0.3.7"]

‎common-app-lib/project.clj‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -28,14 +28,16 @@
2828
:profiles {:security {:plugins [[com.livingsocial/lein-dependency-check "1.4.1"]]
2929
:dependency-check {:output-format [:all]
3030
:suppression-file "resources/security/suppression.xml"}}
31-
:dev {:dependencies [[org.clojure/tools.namespace "0.2.11"]
31+
:dev {:dependencies [[org.bouncycastle/bcpkix-jdk18on "1.85"]
32+
[org.clojure/tools.namespace "0.2.11"]
3233
[org.clojure/tools.nrepl "0.2.13"]
34+
[org.clojars.gjahad/debug-repl "0.3.3"]
3335
[org.eclipse.jetty.ee9/jetty-ee9-servlet "12.1.10"]
34-
[ring/ring-codec "1.3.0"]
36+
[ring/ring-codec "1.3.0"
37+
:exclusions [org.bouncycastle/bcpkix-jdk15on]]
3538
[ring/ring-jetty-adapter "1.15.4"
3639
:exclusions [org.eclipse.jetty.ee9/jetty-ee9-servlet]]
37-
[ring/ring-core "1.15.4"]
38-
[org.clojars.gjahad/debug-repl "0.3.3"]]
40+
[ring/ring-core "1.15.4"]]
3941
:jvm-opts ^:replace ["-server"]
4042
:source-paths ["src" "dev" "test"]}
4143
:static {}

‎dev-system/compose.yaml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ services:
1111
build:
1212
context: .
1313
dockerfile: resources/elasticsearch/Dockerfile.elasticsearch
14-
image: cmr-elasticsearch:8.19.17
14+
image: cmr-elasticsearch:8.19.14
1515
ports:
1616
- 9211:9200
1717
- 9300:9300
@@ -37,7 +37,7 @@ services:
3737
build:
3838
context: .
3939
dockerfile: resources/elasticsearch/Dockerfile.elasticsearch
40-
image: cmr-elasticsearch:8.19.17
40+
image: cmr-elasticsearch:8.19.14
4141
ports:
4242
- 9210:9200
4343
- 9301:9300
@@ -62,7 +62,7 @@ services:
6262
container_name: cmr-kibana
6363
depends_on:
6464
- elasticsearch
65-
image: docker.elastic.co/kibana/kibana:8.19.17
65+
image: docker.elastic.co/kibana/kibana:8.19.14
6666
ports:
6767
- 5602:5601
6868
environment:
@@ -76,7 +76,7 @@ services:
7676
container_name: cmr-gran-kibana
7777
depends_on:
7878
- gran-elasticsearch
79-
image: docker.elastic.co/kibana/kibana:8.19.17
79+
image: docker.elastic.co/kibana/kibana:8.19.14
8080
ports:
8181
- 5601:5601
8282
environment:

‎dev-system/project.clj‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,9 +49,11 @@
4949
:license {:name "Eclipse Public License"
5050
:url "http://www.eclipse.org/legal/epl-v10.html"}
5151
:dependencies ~(concat '[[commons-codec/commons-codec "1.11"]
52+
;; replaces org.bouncycastle/bcpkix-jdk15on
53+
[org.bouncycastle/bcpkix-jdk18on "1.85"]
5254
[org.clojure/clojure "1.11.2"]
5355
[org.eclipse.jetty.ee9/jetty-ee9-servlet "12.1.10"]
54-
[ring/ring-codec "1.3.0"]
56+
[ring/ring-codec "1.3.0" :exclusions [org.bouncycastle/bcpkix-jdk15on]]
5557
[ring/ring-jetty-adapter "1.15.4"
5658
:exclusions [org.eclipse.jetty.ee9/jetty-ee9-servlet]]]
5759
project-dependencies)

‎elastic-utils-lib/project.clj‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
1-
(def elastic-version "8.19.17")
2-
1+
(def elastic-version "8.19.14")
32

43
(defproject nasa-cmr/cmr-elastic-utils-lib "0.1.0-SNAPSHOT"
54
:description "A library containing utilities for dealing with Elasticsearch."

‎elastic-utils-lib/src/cmr/elastic_utils/search/es_index.clj‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -217,7 +217,8 @@
217217
:payload-too-large
218218
"The search is creating more buckets than allowed by CMR. Please narrow your search."))
219219

220-
(when (re-find #"maxClauseCount is set to 1024" body)
220+
(when (and (= 400 (:status (ex-data e)))
221+
(re-find #"(?i)(?:maxClauseCount is set to [0-9]+|too many clauses)" body))
221222
(errors/throw-service-error
222223
:payload-too-large
223224
"The search is creating more clauses than allowed by CMR. Please narrow your search."))

‎elastic-utils-lib/test/cmr/elastic_utils/test/es_index.clj‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,53 @@
33
(:require
44
[clojure.test :refer [deftest is testing]]
55
[cmr.common.services.search.query-model :as qm]
6+
[cmr.elastic-utils.es-helper :as es-helper]
67
[cmr.elastic-utils.search.es-group-query-conditions :as gc]
78
[cmr.elastic-utils.search.es-index :as es-index]))
89

10+
(def clause-limit-message
11+
"The search is creating more clauses than allowed by CMR. Please narrow your search.")
12+
13+
(defn- send-query-with-es-error
14+
[status body]
15+
(with-redefs [es-index/context->conn (constantly nil)
16+
es-helper/search (fn [& _]
17+
(throw (ex-info "Elasticsearch request failed"
18+
{:status status
19+
:body body})))]
20+
(#'es-index/do-send-with-retry
21+
{}
22+
{:index-name "test-index" :type-name "collection"}
23+
{}
24+
1)))
25+
26+
(deftest clause-limit-errors-are-payload-too-large
27+
(doseq [[description body]
28+
[["Elasticsearch 7 default clause limit error"
29+
"maxClauseCount is set to 1024"]
30+
["Elasticsearch 7 configured clause limit error"
31+
"maxClauseCount is set to 4096"]
32+
["Elasticsearch 8 clause limit error"
33+
(str "{\"error\":{\"root_cause\":[{\"type\":\"illegal_argument_exception\","
34+
"\"reason\":\"Query rewrite failed: too many clauses\"}]},\"status\":400}")]]]
35+
(testing description
36+
(let [exception (try
37+
(send-query-with-es-error 400 body)
38+
nil
39+
(catch clojure.lang.ExceptionInfo e
40+
e))]
41+
(is (= :payload-too-large (:type (ex-data exception))))
42+
(is (= [clause-limit-message] (:errors (ex-data exception))))))))
43+
44+
(deftest clause-limit-errors-require-bad-request-status
45+
(let [exception (try
46+
(send-query-with-es-error 500 "Query rewrite failed: too many clauses")
47+
nil
48+
(catch clojure.lang.ExceptionInfo e
49+
e))]
50+
(is (nil? (:type (ex-data exception))))
51+
(is (= 500 (:status (ex-data (ex-cause exception)))))))
52+
953
(deftest test-query->execution-params
1054
(let [query->execution-params #'es-index/query->execution-params
1155
condition (gc/or-conds (map #(qm/string-conditions :consortiums [%])

‎es-spatial-plugin/README.md‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ To package the spatial script for use in elastic:
2727

2828
This will create a zip in target/ ready for installation in elasticsearch.
2929

30-
## ES Spatial Lib with Elastic 8.19.17 changes and Java 17
30+
## ES Spatial Lib with Elastic 8.19.14 changes and Java 17
3131

3232
Due to changes in 8.x requirements for plugins, we updated the es-spatial-lib plugin in the following ways to be able to work with 8.x
3333

@@ -85,12 +85,12 @@ Here are the instructions on how to create the zip and install on your remote ES
8585
- Search for that record using a spatial extent search ( a spatial extent search will invoke the spatial plugin.) You should get your search
8686
response back without crashing elastic search.
8787

88-
### docker-compose.yml for ES 8.19.17 for GRAN CLUSTER for local runs
88+
### docker-compose.yml for ES 8.19.14 for GRAN CLUSTER for local runs
8989

9090
```
9191
services:
9292
elasticsearch:
93-
image: elasticsearch:8.19.17
93+
image: elasticsearch:8.19.14
9494
ports:
9595
- 9210:9200
9696
- 9300:9300
@@ -109,7 +109,7 @@ services:
109109
container_name: cmr-gran-kibana
110110
depends_on:
111111
- elasticsearch
112-
image: docker.elastic.co/kibana/kibana:8.19.17
112+
image: docker.elastic.co/kibana/kibana:8.19.14
113113
ports:
114114
- 5601:5601
115115
environment:
@@ -124,12 +124,12 @@ networks:
124124
elasticsearch-network: {}
125125
```
126126

127-
### docker-compose.yml for ES 8.19.17 for NON-GRAN CLUSTER for local runs
127+
### docker-compose.yml for ES 8.19.14 for NON-GRAN CLUSTER for local runs
128128

129129
```
130130
services:
131131
elasticsearch:
132-
image: elasticsearch:8.19.17
132+
image: elasticsearch:8.19.14
133133
ports:
134134
- 9211:9200
135135
- 9301:9300
@@ -148,7 +148,7 @@ services:
148148
container_name: cmr-kibana
149149
depends_on:
150150
- elasticsearch
151-
image: docker.elastic.co/kibana/kibana:8.19.17
151+
image: docker.elastic.co/kibana/kibana:8.19.14
152152
ports:
153153
- 5602:5601
154154
environment:

0 commit comments

Comments
 (0)