CMR-11452 - Updated default max window to 720 hours (30 days) and added support for header override to allow any size window

Author: jaortega527

bootstrap-app/README.md

@@ -255,6 +255,17 @@ Operator can use the `start_index` parameter to index concepts with sequence num
 The `/after_date_time` endpoint is retained for compatibility. New callers should use
 `/between_date_time`, which bounds the request to an explicit time range.
 Compatibility requests to `/after_date_time` are implicitly bounded from `date_time` to the request timestamp and are rejected when that window exceeds the configured maximum.
+The maximum is 30 days by default.
+
+To allow an authorized request to exceed the configured maximum time range:
+
+    curl -i \
+        -X POST \
+        -H "CMR-Bulk-Index-Ignore-Time-Range-Limit: true" \
+        "http://localhost:3006/bulk_index/after_date_time?date_time=2015-02-02T10:00:00Z"
+
+Only a case-insensitive value of `true` disables the time-range limit. If the header is omitted or
+has any other value, the configured maximum remains enforced.
 
 For all providers and all system concepts:
 

bootstrap-app/src/cmr/bootstrap/api/bulk_index.clj

@@ -2,6 +2,7 @@
   "Defines the bulk index functions for the bootstrap API."
   (:require
    [clj-time.core :as time]
+   [clojure.string :as string]
    [cmr.bootstrap.config :as bootstrap-config]
    [cmr.bootstrap.api.messages :as msg]
    [cmr.bootstrap.api.util :as api-util]
@@ -10,6 +11,9 @@
    [cmr.common.services.errors :as errors]
    [cmr.common.time-keeper :as time-keeper]))
 
+(def ^:private ignore-after-date-time-limit-header
+  "cmr-bulk-index-ignore-time-range-limit")
+
 (defn- parse-date-time-param
   [_param-name date-time]
   (if-let [date-time-value (date-time-parser/try-parse-datetime date-time)]
@@ -77,6 +81,12 @@
        :invalid-data
        (msg/after-date-time-window-exceeded max-window-hours)))))
 
+(defn- ignore-after-date-time-limit?
+  [headers]
+  (= "true"
+     (some-> (get headers ignore-after-date-time-limit-header)
+             string/lower-case)))
+
 (defn index-provider
   "Index all the collections and granules for a given provider."
   [context provider-id-map params]
@@ -112,20 +122,23 @@
 
 (defn data-later-than-date-time
   "Index all data with a revision-date later than the given date-time, with the upper bound set to the request time."
-  [context body params]
-  (let [dispatcher (api-util/get-dispatcher context params :index-data-between-date-time)
-        provider-ids (get body "provider_ids")
-        date-time (:date_time params)
-        start-date-time (parse-date-time-param :date_time date-time)
-        end-date-time (time-keeper/now)]
-    (validate-date-time-range start-date-time end-date-time)
-    (validate-after-date-time-window start-date-time end-date-time)
-    {:status 202
-     :body {:message (msg/data-later-than-date-time
-                      params
-                      (service/index-data-between-date-time
-                       context dispatcher provider-ids start-date-time end-date-time)
-                      date-time)}}))
+  ([context body params]
+   (data-later-than-date-time context body params {}))
+  ([context body params headers]
+   (let [dispatcher (api-util/get-dispatcher context params :index-data-between-date-time)
+         provider-ids (get body "provider_ids")
+         date-time (:date_time params)
+         start-date-time (parse-date-time-param :date_time date-time)
+         end-date-time (time-keeper/now)]
+     (validate-date-time-range start-date-time end-date-time)
+     (when-not (ignore-after-date-time-limit? headers)
+       (validate-after-date-time-window start-date-time end-date-time))
+     {:status 202
+      :body {:message (msg/data-later-than-date-time
+                       params
+                       (service/index-data-between-date-time
+                        context dispatcher provider-ids start-date-time end-date-time)
+                       date-time)}})))
 
 (defn data-between-date-time
   "Index data with revision-date in the requested date-time range.

bootstrap-app/src/cmr/bootstrap/api/routes.clj

@@ -61,9 +61,9 @@
        (POST "/collections" {:keys [request-context body params]}
          (acl/verify-ingest-management-permission request-context :update)
          (bulk-index/index-collection request-context body params))
-       (POST "/after_date_time" {:keys [request-context body params]}
+       (POST "/after_date_time" {:keys [request-context body params headers]}
          (acl/verify-ingest-management-permission request-context :update)
-         (bulk-index/data-later-than-date-time request-context body params))
+         (bulk-index/data-later-than-date-time request-context body params headers))
        (POST "/between_date_time" {:keys [request-context body params]}
          (acl/verify-ingest-management-permission request-context :update)
          (bulk-index/data-between-date-time request-context body params))

bootstrap-app/src/cmr/bootstrap/config.clj

@@ -49,7 +49,7 @@
 (defconfig bulk-index-after-date-time-max-window-hours
   "Maximum number of hours allowed for /bulk_index/after_date_time. Larger windows should use
   /bulk_index/between_date_time explicitly."
-  {:default 168
+  {:default 720
    :type Long})
 
 (declare initialize-kms-on-boot)

bootstrap-app/test/cmr/bootstrap/test/api/bulk_index_test.clj

@@ -204,6 +204,42 @@
                 {}
                 {:date_time "2026-05-13T01:00:00Z"})))))))
 
+(deftest data-later-than-date-time-supports-time-range-limit-override
+  (let [context {:system :system}
+        service-call (atom nil)]
+    (with-redefs [api-util/get-dispatcher (constantly :dispatcher)
+                  time-keeper/now (constantly (time/date-time 2026 5 13 5 0))
+                  bootstrap-config/bulk-index-after-date-time-max-window-hours (constantly 3)
+                  service/index-data-between-date-time
+                  (fn [& args]
+                    (reset! service-call args)
+                    {:message "indexed"})]
+      (testing "When the override header is true, then a range over the configured limit is accepted"
+        (is (= 202
+               (:status
+                (bulk-index/data-later-than-date-time
+                 context
+                 {"provider_ids" ["PROV1"]}
+                 {:date_time "2026-05-13T01:00:00Z"}
+                 {"cmr-bulk-index-ignore-time-range-limit" "TRUE"}))))
+        (is (= [context
+                :dispatcher
+                ["PROV1"]
+                (time/date-time 2026 5 13 1 0)
+                (time/date-time 2026 5 13 5 0)]
+               @service-call)))
+
+      (testing "When the override header is not true, then the configured range limit is enforced"
+        (doseq [header-value ["false" "invalid"]]
+          (is (= :invalid-data
+                 (:type
+                  (service-error
+                   #(bulk-index/data-later-than-date-time
+                     context
+                     {}
+                     {:date_time "2026-05-13T01:00:00Z"}
+                     {"cmr-bulk-index-ignore-time-range-limit" header-value}))))))))))
+
 (deftest data-later-than-date-time-validates-range
   (let [context {:system :system}]
     (with-redefs [api-util/get-dispatcher (constantly :dispatcher)

system-int-test/test/cmr/system_int_test/bootstrap/bulk_index/concepts_test.clj

@@ -318,13 +318,13 @@
 (deftest ^:oracle bulk-index-after-date-time-rejects-large-window
   (s/only-with-real-database
    (try
-     (dev-sys-util/freeze-time! "3016-01-10T00:00:00Z")
+     (dev-sys-util/freeze-time! "3016-02-01T00:00:00Z")
      (let [{:keys [status errors]} (bootstrap/bulk-index-after-date-time
-                                    "3016-01-02T00:00:00Z"
+                                    "3016-01-01T00:00:00Z"
                                     {tc/token-header (tc/echo-system-token)}
                                     ["PROV1"])]
        (is (= 422 status))
-       (is (re-find #"The requested time window exceeds the /bulk_index/after_date_time limit of 168 hours"
+       (is (re-find #"The requested time window exceeds the /bulk_index/after_date_time limit of 720 hours"
                     (first errors))))
      (finally
        (dev-sys-util/clear-current-time!)))))