CMR-11452 - Remove hour break down in bulk-index/after-date-time & bulk-index/between-date-time to improve performance (#2487)

* CMR-11452 - Removed hour "chunking" from bulk-index/after-date-time and bulk-index/between-date-time

* CMR-11452 - Updated default max window to 720 hours (30 days) and added support for header override to allow any size window

* CMR-11452 - Small improvement to error message when after_date_time range is too large

* CMR-11452 - Update header retrieval to use parseBoolean instead of using string comparison

Author: jaortega527

bootstrap-app/README.md

@@ -253,8 +253,19 @@ Operator can use the `start_index` parameter to index concepts with sequence num
 ### Bulk index concepts newer than a given date-time
 
 The `/after_date_time` endpoint is retained for compatibility. New callers should use
-`/between_date_time`, which bounds the request and splits large ranges into smaller indexing chunks.
+`/between_date_time`, which bounds the request to an explicit time range.
 Compatibility requests to `/after_date_time` are implicitly bounded from `date_time` to the request timestamp and are rejected when that window exceeds the configured maximum.
+The maximum is 30 days by default.
+
+To allow an authorized request to exceed the configured maximum time range:
+
+    curl -i \
+        -X POST \
+        -H "CMR-Bulk-Index-Ignore-Time-Range-Limit: true" \
+        "http://localhost:3006/bulk_index/after_date_time?date_time=2015-02-02T10:00:00Z"
+
+Only a case-insensitive value of `true` disables the time-range limit. If the header is omitted or
+has any other value, the configured maximum remains enforced.
 
 For all providers and all system concepts:
 
@@ -295,8 +306,8 @@ To provide a range in hours instead of an explicit end date-time:
         "http://localhost:3006/bulk_index/between_date_time?start_date_time=2015-02-02T10:00:00Z&hours=2"
 
 The `start_date_time` parameter is required. Callers can provide either `end_date_time` or `hours`.
-If neither is provided, bootstrap uses the end of the start date's day. Internally, bootstrap enforces
-smaller chunks across the requested time range before publishing indexing work.
+If neither is provided, bootstrap uses the end of the start date's day. Bootstrap publishes one
+indexing request per provider covering the complete requested time range.
 
 ### Bulk index all system concepts (tags/acls/access-groups)
 

bootstrap-app/src/cmr/bootstrap/api/bulk_index.clj

@@ -10,6 +10,9 @@
    [cmr.common.services.errors :as errors]
    [cmr.common.time-keeper :as time-keeper]))
 
+(def ^:private ignore-after-date-time-limit-header
+  "cmr-bulk-index-ignore-time-range-limit")
+
 (defn- parse-date-time-param
   [_param-name date-time]
   (if-let [date-time-value (date-time-parser/try-parse-datetime date-time)]
@@ -77,6 +80,11 @@
        :invalid-data
        (msg/after-date-time-window-exceeded max-window-hours)))))
 
+(defn- ignore-after-date-time-limit?
+  [headers]
+  (Boolean/parseBoolean
+   (get headers ignore-after-date-time-limit-header)))
+
 (defn index-provider
   "Index all the collections and granules for a given provider."
   [context provider-id-map params]
@@ -112,20 +120,23 @@
 
 (defn data-later-than-date-time
   "Index all data with a revision-date later than the given date-time, with the upper bound set to the request time."
-  [context body params]
-  (let [dispatcher (api-util/get-dispatcher context params :index-data-between-date-time)
-        provider-ids (get body "provider_ids")
-        date-time (:date_time params)
-        start-date-time (parse-date-time-param :date_time date-time)
-        end-date-time (time-keeper/now)]
-    (validate-date-time-range start-date-time end-date-time)
-    (validate-after-date-time-window start-date-time end-date-time)
-    {:status 202
-     :body {:message (msg/data-later-than-date-time
-                      params
-                      (service/index-data-between-date-time
-                       context dispatcher provider-ids start-date-time end-date-time)
-                      date-time)}}))
+  ([context body params]
+   (data-later-than-date-time context body params {}))
+  ([context body params headers]
+   (let [dispatcher (api-util/get-dispatcher context params :index-data-between-date-time)
+         provider-ids (get body "provider_ids")
+         date-time (:date_time params)
+         start-date-time (parse-date-time-param :date_time date-time)
+         end-date-time (time-keeper/now)]
+     (validate-date-time-range start-date-time end-date-time)
+     (when-not (ignore-after-date-time-limit? headers)
+       (validate-after-date-time-window start-date-time end-date-time))
+     {:status 202
+      :body {:message (msg/data-later-than-date-time
+                       params
+                       (service/index-data-between-date-time
+                        context dispatcher provider-ids start-date-time end-date-time)
+                       date-time)}})))
 
 (defn data-between-date-time
   "Index data with revision-date in the requested date-time range.

bootstrap-app/src/cmr/bootstrap/api/messages.clj

@@ -98,7 +98,7 @@
 (defn after-date-time-window-exceeded
   [max-window-hours]
   (format (str "The requested time window exceeds the /bulk_index/after_date_time limit of %d hours. "
-               "Please use a smaller date_time value so the range to now is within %d hours.")
+               "Please use a later date_time value so the range to now is within %d hours.")
           max-window-hours
           max-window-hours))
 

bootstrap-app/src/cmr/bootstrap/api/routes.clj

@@ -61,9 +61,9 @@
        (POST "/collections" {:keys [request-context body params]}
          (acl/verify-ingest-management-permission request-context :update)
          (bulk-index/index-collection request-context body params))
-       (POST "/after_date_time" {:keys [request-context body params]}
+       (POST "/after_date_time" {:keys [request-context body params headers]}
          (acl/verify-ingest-management-permission request-context :update)
-         (bulk-index/data-later-than-date-time request-context body params))
+         (bulk-index/data-later-than-date-time request-context body params headers))
        (POST "/between_date_time" {:keys [request-context body params]}
          (acl/verify-ingest-management-permission request-context :update)
          (bulk-index/data-between-date-time request-context body params))

bootstrap-app/src/cmr/bootstrap/config.clj

@@ -46,15 +46,10 @@
   {:default 1
    :type Long})
 
-(defconfig bulk-index-between-date-time-window-hours
-  "Maximum number of hours for each /bulk_index/between_date_time indexing chunk."
-  {:default 1
-   :type Long})
-
 (defconfig bulk-index-after-date-time-max-window-hours
   "Maximum number of hours allowed for /bulk_index/after_date_time. Larger windows should use
   /bulk_index/between_date_time explicitly."
-  {:default 168
+  {:default 720
    :type Long})
 
 (declare initialize-kms-on-boot)

bootstrap-app/src/cmr/bootstrap/services/dispatch/impl/message_queue.clj

@@ -2,7 +2,6 @@
   "Functions implementing the dispatch protocol to support bootstrap operations using a message
   queue."
   (:require
-   [clj-time.core :as time]
    [cmr.bootstrap.api.messages-bulk-index :as msg]
    [cmr.bootstrap.config :as config]
    [cmr.bootstrap.data.bulk-index :as bulk-index]
@@ -123,40 +122,22 @@
        (message-queue/bootstrap-provider-event provider-id nil date-time)))
     (info (msg/index-data-later-than-date-time-done))))
 
-(defn- date-time-chunks
-  "Returns half-open date-time ranges no larger than chunk-hours."
-  [start-date-time end-date-time chunk-hours]
-  (let [chunk-period (time/hours chunk-hours)]
-    (loop [chunk-start start-date-time
-           chunks []]
-      (if (time/before? chunk-start end-date-time)
-        (let [candidate-end (time/plus chunk-start chunk-period)
-              chunk-end (if (time/before? candidate-end end-date-time)
-                          candidate-end
-                          end-date-time)]
-          (recur chunk-end (conj chunks [chunk-start chunk-end])))
-        chunks))))
-
 (defn- index-data-between-date-time
   "Bulk index all the concepts with revision dates between the given date-times."
   [_this context provider-ids start-date-time end-date-time]
   (let [provider-ids (if (seq provider-ids)
                        provider-ids
                        ;; all providers including CMR provider which is for system concepts
-                       (conj (map :provider-id (helper/get-providers (:system context))) "CMR"))
-        chunk-hours (max 1 (config/bulk-index-between-date-time-window-hours))
-        chunks (date-time-chunks start-date-time end-date-time chunk-hours)]
-    (doseq [provider-id provider-ids
-            [chunk-start chunk-end] chunks]
+                       (conj (map :provider-id (helper/get-providers (:system context))) "CMR"))]
+    (doseq [provider-id provider-ids]
       (message-queue/publish-bootstrap-concepts-event
        context
        (message-queue/bootstrap-provider-between-date-time-event
-        provider-id chunk-start chunk-end)))
-    (info (format "Published %d bulk index messages between [%s] and [%s] using %d hour chunks."
-                  (* (count provider-ids) (count chunks))
+        provider-id start-date-time end-date-time)))
+    (info (format "Published %d bulk index messages between [%s] and [%s], one per provider."
+                  (count provider-ids)
                   start-date-time
-                  end-date-time
-                  chunk-hours))))
+                  end-date-time))))
 
 (defn- fingerprint-variables
   "Update fingerprints of variables. If a provider is passed, only update fingerprints of the

bootstrap-app/test/cmr/bootstrap/test/api/bulk_index_test.clj

@@ -197,13 +197,49 @@
                   bootstrap-config/bulk-index-after-date-time-max-window-hours (constantly 3)]
       (is (= {:type :invalid-data
               :errors [(str "The requested time window exceeds the /bulk_index/after_date_time limit of 3 hours. "
-                            "Please use a smaller date_time value so the range to now is within 3 hours.")]}
+                            "Please use a later date_time value so the range to now is within 3 hours.")]}
              (service-error
               #(bulk-index/data-later-than-date-time
                 context
                 {}
                 {:date_time "2026-05-13T01:00:00Z"})))))))
 
+(deftest data-later-than-date-time-supports-time-range-limit-override
+  (let [context {:system :system}
+        service-call (atom nil)]
+    (with-redefs [api-util/get-dispatcher (constantly :dispatcher)
+                  time-keeper/now (constantly (time/date-time 2026 5 13 5 0))
+                  bootstrap-config/bulk-index-after-date-time-max-window-hours (constantly 3)
+                  service/index-data-between-date-time
+                  (fn [& args]
+                    (reset! service-call args)
+                    {:message "indexed"})]
+      (testing "When the override header is true, then a range over the configured limit is accepted"
+        (is (= 202
+               (:status
+                (bulk-index/data-later-than-date-time
+                 context
+                 {"provider_ids" ["PROV1"]}
+                 {:date_time "2026-05-13T01:00:00Z"}
+                 {"cmr-bulk-index-ignore-time-range-limit" "TRUE"}))))
+        (is (= [context
+                :dispatcher
+                ["PROV1"]
+                (time/date-time 2026 5 13 1 0)
+                (time/date-time 2026 5 13 5 0)]
+               @service-call)))
+
+      (testing "When the override header is not true, then the configured range limit is enforced"
+        (doseq [header-value ["false" "invalid"]]
+          (is (= :invalid-data
+                 (:type
+                  (service-error
+                   #(bulk-index/data-later-than-date-time
+                     context
+                     {}
+                     {:date_time "2026-05-13T01:00:00Z"}
+                     {"cmr-bulk-index-ignore-time-range-limit" header-value}))))))))))
+
 (deftest data-later-than-date-time-validates-range
   (let [context {:system :system}]
     (with-redefs [api-util/get-dispatcher (constantly :dispatcher)

bootstrap-app/test/cmr/bootstrap/test/services/dispatch/impl/message_queue_test.clj

@@ -2,30 +2,12 @@
   (:require
    [clj-time.core :as time]
    [clojure.test :refer [deftest is testing]]
-   [cmr.bootstrap.config :as config]
    [cmr.bootstrap.data.bulk-index :as bulk-index-data]
    [cmr.bootstrap.data.message-queue :as message-queue]
    [cmr.bootstrap.embedded-system-helper :as helper]
    [cmr.bootstrap.services.dispatch.core :as dispatch]
    [cmr.bootstrap.services.dispatch.impl.message-queue :as message-queue-dispatcher]))
 
-(deftest date-time-chunks-creates-half-open-ranges
-  (let [start (time/date-time 2026 5 13 1 0)
-        end (time/date-time 2026 5 13 4 30)]
-    (is (= [[(time/date-time 2026 5 13 1 0)
-             (time/date-time 2026 5 13 2 0)]
-            [(time/date-time 2026 5 13 2 0)
-             (time/date-time 2026 5 13 3 0)]
-            [(time/date-time 2026 5 13 3 0)
-             (time/date-time 2026 5 13 4 0)]
-            [(time/date-time 2026 5 13 4 0)
-             (time/date-time 2026 5 13 4 30)]]
-           (#'message-queue-dispatcher/date-time-chunks start end 1)))))
-
-(deftest date-time-chunks-returns-empty-when-range-is-empty
-  (let [start (time/date-time 2026 5 13 1 0)]
-    (is (= [] (#'message-queue-dispatcher/date-time-chunks start start 1)))))
-
 (deftest bootstrap-provider-between-date-time-event-test
   (let [start (time/date-time 2026 5 13 1 0)
         end (time/date-time 2026 5 13 2 0)]
@@ -38,34 +20,31 @@
             start
             end)))))
 
-(deftest index-data-between-date-time-publishes-provider-chunks
+(deftest index-data-between-date-time-publishes-full-range-per-provider
   (let [published (atom [])
-        context {:system {:providers [{:provider-id "PROV1"}]}}
+        context {:system {:providers [{:provider-id "PROV1"}
+                                      {:provider-id "PROV2"}]}}
         start (time/date-time 2026 5 13 1 0)
         end (time/date-time 2026 5 13 3 30)]
-    (with-redefs [config/bulk-index-between-date-time-window-hours (constantly 1)
-                  message-queue/publish-bootstrap-concepts-event
+    (with-redefs [message-queue/publish-bootstrap-concepts-event
                   (fn [_context msg]
                     (swap! published conj msg))]
-      (dispatch/index-data-between-date-time
-       (message-queue-dispatcher/->MessageQueueDispatcher)
-       context
-       ["PROV1"]
-       start
-       end)
-      (is (= [{:action :index-provider-between-date-time
-               :provider-id "PROV1"
-               :start-date-time (time/date-time 2026 5 13 1 0)
-               :end-date-time (time/date-time 2026 5 13 2 0)}
-              {:action :index-provider-between-date-time
-               :provider-id "PROV1"
-               :start-date-time (time/date-time 2026 5 13 2 0)
-               :end-date-time (time/date-time 2026 5 13 3 0)}
-              {:action :index-provider-between-date-time
-               :provider-id "PROV1"
-               :start-date-time (time/date-time 2026 5 13 3 0)
-               :end-date-time (time/date-time 2026 5 13 3 30)}]
-             @published)))))
+      (testing "When provider IDs are supplied, then one full-range event is published per provider"
+        (dispatch/index-data-between-date-time
+         (message-queue-dispatcher/->MessageQueueDispatcher)
+         context
+         ["PROV1" "PROV2"]
+         start
+         end)
+        (is (= [{:action :index-provider-between-date-time
+                 :provider-id "PROV1"
+                 :start-date-time (time/date-time 2026 5 13 1 0)
+                 :end-date-time (time/date-time 2026 5 13 3 30)}
+                {:action :index-provider-between-date-time
+                 :provider-id "PROV2"
+                 :start-date-time (time/date-time 2026 5 13 1 0)
+                 :end-date-time (time/date-time 2026 5 13 3 30)}]
+               @published))))))
 
 (deftest index-data-between-date-time-expands-empty-provider-list
   (let [published (atom [])
@@ -74,11 +53,10 @@
         end (time/date-time 2026 5 13 2 0)]
     (with-redefs [helper/get-providers (constantly [{:provider-id "PROV1"}
                                                     {:provider-id "PROV2"}])
-                  config/bulk-index-between-date-time-window-hours (constantly 0)
                   message-queue/publish-bootstrap-concepts-event
                   (fn [_context msg]
                     (swap! published conj msg))]
-      (testing "empty provider ids uses all providers plus CMR and clamps chunk hours to one"
+      (testing "When provider IDs are omitted, then one full-range event is published for all providers plus CMR"
         (dispatch/index-data-between-date-time
          (message-queue-dispatcher/->MessageQueueDispatcher)
          context

system-int-test/test/cmr/system_int_test/bootstrap/bulk_index/concepts_test.clj

@@ -318,13 +318,13 @@
 (deftest ^:oracle bulk-index-after-date-time-rejects-large-window
   (s/only-with-real-database
    (try
-     (dev-sys-util/freeze-time! "3016-01-10T00:00:00Z")
+     (dev-sys-util/freeze-time! "3016-02-01T00:00:00Z")
      (let [{:keys [status errors]} (bootstrap/bulk-index-after-date-time
-                                    "3016-01-02T00:00:00Z"
+                                    "3016-01-01T00:00:00Z"
                                     {tc/token-header (tc/echo-system-token)}
                                     ["PROV1"])]
        (is (= 422 status))
-       (is (re-find #"The requested time window exceeds the /bulk_index/after_date_time limit of 168 hours"
+       (is (re-find #"The requested time window exceeds the /bulk_index/after_date_time limit of 720 hours"
                     (first errors))))
      (finally
        (dev-sys-util/clear-current-time!)))))