CMR-10529 visualisation search improvments (#2263)

* Revert "CMR-10533: Bumping up elastic to a recomended point release (#2246)"
* CMR-10529: Visualization indexing improvments, fixing Keyword search to correctly include data from multiple fields by adding two new features to Generics, you can append keywords and fields with the same name

Author: jceaser

Generics.md

@@ -5,6 +5,30 @@ Generic documents are documents which conform to the "Generic" API in CMR. These
 
 ## Configuration
 
+### Generic Configuration File
+
+Within the Generic config.json file there is a section called `IndexConfiguration`. This contains two settings:
+
+* AllowAppending: When set to true, then multiple Indexes with the same `Name` value will be appended together to create one larger value. Otherwise the last one in the config file will be the setting used.
+* AdditionalKeywords: List of simple fields to be added to the `keyword` field for general searching. By default CMR uses: LongName, Version, Description, RelatedURLs.
+
+Indexes:
+* Description: Human readable description of field. Shows up in some logs
+* Field: [jq](https://jqlang.org) like path to field data
+* Name: Field Name
+* Mapping: The Elastic field type
+	* token: text-field-mapping
+	* string: string-field-mapping
+	* int: int-field-mapping
+	* date: date-field-mapping
+* Indexer:
+	* default (none): direct, one-to-one mapping
+	* simple-array-field: index a sub field of an array element
+	* complex-fields-only: Complex indexer field that handles both single objects and arrays, and formats using field values only (not field names)
+	* complex-field: takes a list of sub fields and combines them
+
+### CMR settings
+
 If adding a new document, you will need to update the defconf variable by either setting an ENV for global change, or by updating the default value in [/common-lib/src/cmr/common/config.clj](/common-lib/src/cmr/common/config.clj). The format for this value is either JSON for an ENV variable or a clojure map if setting directly in the default attribute of the defconfig like this:
 
 	(defconfig approved-pipeline-documents

elastic-utils-lib/src/cmr/elastic_utils/generics.clj

@@ -0,0 +1,12 @@
+(ns cmr.elastic-utils.generics
+  "A set of functions for dealing with Generics and Elastic")
+
+(defn only-elastic-preferences
+  "Go through all the index configurations and return only the ones related to
+   generating elastic values. If an index does not specify what type it is for,
+   then assume elastic"
+  [list-of-indexes]
+  (keep #(if (not (nil? %)) %)
+        (map
+         (fn [x] (when (or (nil? (:Type x)) (= "elastic" (:Type x))) x))
+         list-of-indexes)))

indexer-app/src/cmr/indexer/data/concepts/generic.clj

@@ -9,15 +9,17 @@
    [clojure.string :as string]
    [cmr.common.concepts :as concepts]
    [cmr.common.generics :as generics]
+   [cmr.common.time-keeper :as time-keeper]
    [cmr.common.util :as util]
+   [cmr.common-app.config :as app-config]
    [cmr.indexer.data.concepts.association-util :as assoc-util]
    [cmr.indexer.data.concept-parser :as c-parser]
    [cmr.indexer.data.concepts.generic-util :as gen-util]
    [cmr.indexer.data.concepts.keyword-util :as keyword-util]
    [cmr.indexer.data.elasticsearch :as esearch]
    [cmr.transmit.metadata-db :as meta-db]))
 
-(defn field->index-complex-field
+(defn- field->index-complex-field
   "This is an example of a complex indexer which takes a list of sub fields and
   combines them into one field"
   [settings data]
@@ -39,7 +41,7 @@
     {(keyword field-name) field-value
      (keyword field-name-lower) field-value-lower}))
 
-(defn field->index-complex-field-with-values-only
+(defn- field->index-complex-field-with-values-only
   "Complex indexer field that handles both single objects and arrays,
    and formats using field values only (not field names)"
   [settings data]
@@ -56,7 +58,7 @@
                               (let [values (map #(get element (keyword %)) sub-fields)]
                                 (apply format layout values)))
                             field-data)
-                      ;; Handle single object case  
+                      ;; Handle single object case
                       (let [values (map #(get field-data (keyword %)) sub-fields)]
                         (apply format layout values)))
 
@@ -66,37 +68,61 @@
     {(keyword field-name) field-value
      (keyword field-name-lower) field-value-lower}))
 
-(defn field->index-simple-array-field
-  "The gets the a sub field of an array element and puts those values
-  into a list so that each value can be searched on one of two indexes:
-  the literal case, another is all lower case."
+(defn- field->index-simple-array-field
+  "The gets the a sub field of an array element and puts those values into a list so that each value
+   can be searched on one of two indexes: the literal case, another is all lower case.
+   Example usage:
+   (field->index-simple-array-field
+     ;; settings
+     {:Field \".ConceptIds\"
+      :Name \"Concept-Ids\"
+      :Configuration {:sub-fields [\"Value\", \"ShortName\"]}}
+     ;; data
+     {:ConceptIds [{:Value \"V-One\" :ShortName \"Short-One\" :ignore \"I-one\"}
+                   {:Value \"V-Two\" :ShortName \"Short-Two\" :ignore \"I-two\"}]})
+   Result:
+   {:concept-ids \"Short-Two Short-One V-Two V-One\"
+    :concept-ids-lowercase \"short-two short-one v-two v-one\"}
+   "
   [settings data]
   (let [field-list (get settings :Field ".")
         field-data (get-in data (generics/jq->list field-list keyword) {})
         field-name (util/safe-lowercase (:Name settings))
         field-name-lower (str field-name "-lowercase")
         config (get settings :Configuration {})
         sub-fields (get config :sub-fields {})
-        value (reduce (fn [data, key-name]
-                        (into data (map (keyword key-name) field-data)))
-                      (sequence nil)
-                      sub-fields)
-        value-lower (map #(util/safe-lowercase %) value)]
+        value-raw (reduce (fn [data, key-name]
+                            (into data (map (keyword key-name) field-data)))
+                          (sequence nil)
+                          sub-fields)
+        value (cond
+                (string? value-raw) value-raw
+                (coll? value-raw) (clojure.string/join " " (map str value-raw))
+                :else (str value-raw))
+        value-lower (util/safe-lowercase value)]
     {(keyword field-name) value
      (keyword field-name-lower) value-lower}))
 
-(defn field->index-default-field
-  "The default indexer which will map one metadata field to two indexes. One is
-  with the literal case, another is all lower case"
+(defn- field->index-default-field
+  "The default indexer which will map one metadata field to two indexes. One is with the literal
+   case, another is all lower case. If no field is found, then don't return anything
+   Example Usage:
+   (field->index-default-field
+     {:Field \".VisualizationType\" :Name \"VisualizationType\"}
+     {:VisualizationType \"Default\"})
+   Result:
+   {:visualizationtype \"Default\", :visualizationtype-lowercase \"default\"}
+   "
   [settings data]
   (let [field-name (util/safe-lowercase (:Name settings))
         field-name-lower (str field-name "-lowercase")
         value (get-in data (generics/jq->list (:Field settings) keyword))
         value-lower (util/safe-lowercase value)]
-    {(keyword field-name) value
-     (keyword field-name-lower) value-lower}))
+    (when value
+      {(keyword field-name) value
+       (keyword field-name-lower) value-lower})))
 
-(defn field->index
+(defn- field->index
   "Functions which convert a part of metadata to a name-value which can be added
   to an index document. This function is directed by looking for an :Indexer
   value in settings and assuming :default if it is not set.
@@ -116,51 +142,111 @@
     "simple-array-field" (field->index-simple-array-field settings data)
     (field->index-default-field settings data)))
 
+(defn- merge-or-concat
+  "Facilitate the behavior of the :AllowAppending configuration by concatinating two values"
+  ([existing new-value]
+   (merge-or-concat existing new-value " "))
+  ([existing new-value separator]
+     (string/trim (if (nil? existing)
+                    (str new-value)
+                    (if (or (empty? existing) (empty? new-value))
+                      (str existing new-value)
+                      (str existing separator new-value))))))
+
+(defn- post-process-keyword
+  "Go through the :keyword and :keyword-lowercase fields in the Elastic Document and make sure they
+   are tokenized and sorted in the 'CMR' way."
+  [doc]
+  (-> doc
+      (assoc :keyword
+             (keyword-util/field-values->keyword-text [(:keyword doc)]))
+      (assoc :keyword-lowercase
+             (keyword-util/field-values->keyword-text [(:keyword-lowercase doc)]))))
+
+(defn- release-number
+  "Format the software release number for using in the index document. Use a date if in development."
+  []
+  (let [rel-ver (app-config/release-version)]
+    (if (= rel-ver "dev")
+      (format "%s-%s", rel-ver, (time-keeper/now))
+      (app-config/release-version))))
+
 (defn- parsed-concept->elastic-doc
   "Generate elastic document"
-  [context concept parsed-concept]
-  (let [{:keys [concept-id revision-id deleted provider-id user-id
-                revision-date native-id]} concept
-        parsed-concept (if (:deleted concept)
-                         (c-parser/parse-concept context concept)
-                         parsed-concept)
-        generic-associations (esearch/parse-non-tombstone-associations
-                              context
-                              (meta-db/get-generic-associations-for-concept context concept))
-        gen-name (csk/->kebab-case (get-in parsed-concept [:MetadataSpecification :Name] ""))
-        gen-ver (get-in parsed-concept [:MetadataSpecification :Version])
-        index-data-file (format "schemas/%s/v%s/config.json" gen-name gen-ver)
-        index-file-raw (slurp (io/resource index-data-file))
-        index-data (json/parse-string index-file-raw true)
-        schema-keys [:LongName
-                     :Version
-                     :Description
-                     :RelatedURLs]
-        keyword-values (keyword-util/concept-keys->keyword-text
-                        parsed-concept schema-keys)
-        common-doc ;; fields common to all generic documents
-        {:concept-id concept-id
-         :revision-id revision-id
-         :deleted deleted
-         :gen-name gen-name
-         :gen-name-lowercase (util/safe-lowercase gen-name)
-         :gen-version gen-ver
-         :generic-type (str gen-name " " gen-ver)
-         :provider-id provider-id
-         :provider-id-lowercase (util/safe-lowercase provider-id)
-         :keyword keyword-values
-         :user-id user-id
-         :revision-date revision-date
-         :native-id native-id
-         :native-id-lowercase (string/lower-case native-id)
-         :associations-gzip-b64 (assoc-util/associations->gzip-base64-str generic-associations concept-id)}
-        configs (gen-util/only-elastic-preferences (:Indexes index-data))
-        ;; now add the configured indexes
-        doc (reduce
-             (fn [data, config] (into data (field->index config parsed-concept)))
-             common-doc
-             configs)]
-    doc))
+  ([context concept parsed-concept]
+   ;; Abstract away the need to have context or find files in the jar with this form of the function
+   ;; so that the next interface can be called in tests.
+   (let [parsed-concept (if (:deleted concept)
+                          (c-parser/parse-concept context concept)
+                          parsed-concept)
+         generic-associations (esearch/parse-non-tombstone-associations
+                               context
+                               (meta-db/get-generic-associations-for-concept context concept))
+         gen-name (csk/->kebab-case (get-in parsed-concept [:MetadataSpecification :Name] ""))
+         gen-ver (get-in parsed-concept [:MetadataSpecification :Version])
+         index-data (-> "schemas/%s/v%s/config.json"
+                        (format gen-name gen-ver)
+                        (io/resource)
+                        (slurp)
+                        (json/parse-string true))]
+     (parsed-concept->elastic-doc concept
+                                  parsed-concept
+                                  generic-associations
+                                  gen-name
+                                  gen-ver
+                                  index-data)))
+
+  ;; Creating a new overload of the function so that the core functionality can be tested without a
+  ;; context
+  ([concept parsed-concept generic-associations gen-name gen-ver index-data]
+   (let [{:keys [concept-id revision-id deleted provider-id user-id
+                 revision-date native-id]} concept
+         allow-appending (get-in index-data [:IndexConfiguration :AllowAppending])
+         additional-keywords (get-in index-data [:IndexConfiguration :AdditionalKeywords] {})
+         ;; Allow Generic Document Configurations to extend the list of simple top level fields that
+         ;; can be added to the keyword field. This process does not preclude the use of more
+         ;; complicated fields also being added when using the "AllowAppending" setting.
+         schema-keys (into [:LongName
+                            :Version
+                            :Description
+                            :RelatedURLs]
+                           (map keyword additional-keywords))
+         keyword-values (keyword-util/concept-keys->keyword-text
+                         parsed-concept schema-keys)
+         common-doc ;; fields common to all generic documents
+         {:cmr-version (release-number)
+          :concept-id concept-id
+          :revision-id revision-id
+          :deleted deleted
+          :gen-name gen-name
+          :gen-name-lowercase (util/safe-lowercase gen-name)
+          :gen-version gen-ver
+          :generic-type (str gen-name " " gen-ver)
+          :provider-id provider-id
+          :provider-id-lowercase (util/safe-lowercase provider-id)
+          :keyword keyword-values
+          :keyword-lowercase keyword-values
+          :user-id user-id
+          :revision-date revision-date
+          :native-id native-id
+          :native-id-lowercase (string/lower-case native-id)
+          :associations-gzip-b64 (assoc-util/associations->gzip-base64-str
+                                  generic-associations concept-id)}
+         configs (gen-util/only-elastic-preferences (:Indexes index-data))
+         ;; now add the configured indexes
+         doc (reduce
+              (fn [data config]
+                ;; Allow Appending from the configuration file allows for multiple rules to be
+                ;; defined using the same :Name. When this is set, then these indexes are added
+                ;; together to form one larger value made up of different parts of the Generic
+                ;; document. Otherwise, the legacy behavier is that the last rule 'wins'.
+                (if allow-appending
+                  (merge-with merge-or-concat data (field->index config parsed-concept))
+                  (into data (field->index config parsed-concept))))
+              common-doc
+              configs)]
+     ;; Finally clean up the keyword indexs which may have been appended
+     (post-process-keyword doc))))
 
 (defn field->keyword-fields
   "Gets the name of the fields that are used for keyword searches.
@@ -197,15 +283,18 @@
         parsed-concept (if (:deleted concept)
                          (c-parser/parse-concept context concept)
                          parsed-concept)
+        ;; No associations in drafts
         concept-type (concepts/concept-id->type (:concept-id concept))
         version (generics/current-generic-version concept-type)
         gen-name (csk/->kebab-case (get-in parsed-concept [:MetadataSpecification :Name] ""))
         gen-ver (get-in parsed-concept [:MetadataSpecification :Version])
         index-data-file (format "schemas/%s/v%s/config.json" (name concept-type) version)
         index-file-raw (slurp (io/resource index-data-file))
         index-data (json/parse-string index-file-raw true)
+        ;; No default keyword processing till latter
         common-doc ;; fields common to all generic documents
-        {:concept-id concept-id
+        {:cmr-version (release-number)
+         :concept-id concept-id
          :revision-id revision-id
          :deleted deleted
          :gen-name gen-name

indexer-app/src/cmr/indexer/data/concepts/generic_util.clj

@@ -1,16 +1,14 @@
 (ns cmr.indexer.data.concepts.generic-util
   "Contains functions to parse and convert Generic Documents (that is a document
    complying to a schema supported by the Generic Document system) to and object
-   that can be indexed in lucine."
-  (:require
-   [clojure.string :as string]))
+   that can be indexed in lucine.")
 
 (defn only-elastic-preferences
   "Go through all the index configurations and return only the ones related to
    generating elastic values. If an index does not specify what type it is for,
    then assume elastic"
   [list-of-indexes]
-  (keep #(if (not (nil? %)) %)
+  (keep #(when (not (nil? %)) %)
         (map
          (fn [x] (when (or (nil? (:Type x)) (= "elastic" (:Type x))) x))
          list-of-indexes)))

indexer-app/src/cmr/indexer/data/elasticsearch.clj

@@ -3,7 +3,7 @@
    [clj-http.client :as client]
    [cmr.common.concepts :as cs]
    [cmr.common.lifecycle :as lifecycle]
-   [cmr.common.log :as log :refer [info warn error]]
+   [cmr.common.log :as log :refer [info infof warn error]]
    [cmr.common.services.errors :as errors]
    [cmr.common.util :as util]
    [cmr.elastic-utils.connect :as es]
@@ -328,12 +328,12 @@
       (when (:error result)
         (if (= 409 (:status result))
           (if ignore-conflict?
-            (info (str "Ignore conflict: " (str result)))
+            (infof "Ignore conflict: %s" (str result))
             (errors/throw-service-error
              :conflict
-             (str "Save to Elasticsearch failed " (str result))))
+             (format "Save to Elasticsearch failed %s" (str result))))
           (errors/internal-error!
-           (str "Save to Elasticsearch failed " (str result))))))))
+           (format "Save to Elasticsearch failed %s" (str result))))))))
 
 (defn get-document
   "Get the document from Elasticsearch, raise error if failed."
@@ -365,6 +365,6 @@
        (when-not (some #{200 404} [status])
          (if (= 409 status)
            (if ignore-conflict?
-             (info (str "Ignore conflict: " (str response)))
+             (infof "Ignore conflict: %s" (str response))
              (errors/throw-service-error :conflict (str "Delete from Elasticsearch failed " (str response))))
            (errors/internal-error! (str "Delete from Elasticsearch failed " (str response)))))))))

indexer-app/src/cmr/indexer/data/index_set_elasticsearch.clj

@@ -113,7 +113,7 @@
                                              :client-id t-config/cmr-client-id}
                                    :throw-exceptions false})
           status (:status response)]
-      (if-not (some #{200 202 204} [status])
+      (when-not (some #{200 202 204} [status])
         (errors/internal-error! (m/index-delete-failure-msg response))))))
 
 (defn save-document-in-elastic