CMR-10723 bump up urllib to a modern version (#2282)

* bump up urllib to a supported version
* added command line flag for running one job as a test
* output moved to logs
* fixed end point function to construct a localhost url, for testing

Author: jceaser

job-utilities/local_development/.editorconfig

@@ -0,0 +1,10 @@
+[*]
+end_of_line = lf
+indent_size = 2
+indent_style = space
+
+[*.{md,py}]
+indent_size = 4 #this is important, the markdown for API docs will not see 2 spaces
+indent_style = space
+insert_final_newline = true
+trim_trailing_whitespace = true

job-utilities/local_development/README.md

@@ -10,4 +10,4 @@ Cron jobs are assumed to be run once every day at a given time. Most jobs are li
 ### Running
 
 Ensure you have the right dependencies by running `pip3 install -r requirements.txt` and then simply run the local_scheduler.py program.
-Note that the program currently does not run jobs immediately upon scheduling.
+Note that the program currently does not run jobs immediately upon scheduling. To run a job right away as a test, use the -t flag.

job-utilities/local_development/local_scheduler.py

@@ -1,38 +1,54 @@
+#!/usr/bin/env python3
+
 """
 local_scheduler takes the job details json file to create
 a simple schedule with the Python schedule library.
 Interval jobs are simply translated unto the library,
 cron jobs are assumed to run at a given HH:MM every day.
 """
+import argparse
+import logging
 import time
 import json
 import os
+import sys
+
+# pylint: disable=import-error
 import schedule
 import urllib3
 
+# setup logger
+logging.basicConfig(level=logging.INFO,
+                    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+                    filename=f'{__file__}.log')
+logger: logging.Logger = logging.getLogger(__name__)
+
 service_ports_file_name = os.getenv("SERVICE_PORTS_FILE", "service-ports.json")
 job_details_file_name = os.getenv("JOB_DETAILS_FILE", "../job-details.json")
+cmr_host_name: str = os.getenv("CMR_HOST_NAME", "localhost")
 
-pool_manager = urllib3.PoolManager(headers={"Authorization" : "mock-echo-system-token"})
+pool_manager = urllib3.PoolManager(headers={"Authorization" : "mock-echo-system-token",
+                                            "client-id": f'{__file__}'})
 
 with open(service_ports_file_name, encoding="UTF-8") as service_ports_file:
     service_port_map = json.load(service_ports_file)
 
-def build_endpoint(job):
+def build_endpoint(host_name, job):
     """
     Takes the job details and builds the local job endpoint
     """
-    url = "http://{}:{}"
+    url = "http://{}:{}/{}"
     port = service_port_map[job["target"]["service"]]
-    return url.format(port, job["target"]["endpoint"])
+    return url.format(host_name, port, job["target"]["endpoint"])
 
-def run_job(details, name):
+def run_job(job_details: dict, job_name :str):
     """
     Takes the job details and runs a REST request on the job endpoint.
     """
-    print('send ' + details["target"]["request-type"] + \
-          ' to ' + details["target"]["endpoint"] + ' for job ' + name)
-    pool_manager.request(details["target"]["request-type"], build_endpoint(details))
+    logger.info('send ' + job_details["target"]["request-type"] + \
+          ' to ' + job_details["target"]["endpoint"] + ' for job ' + job_name)
+    url: str = build_endpoint(cmr_host_name, job_details)
+    pool_manager.request(job_details["target"]["request-type"], url)
 
 def create_schedule():
     """
@@ -46,22 +62,39 @@ def create_schedule():
             if job_details["scheduling"]["type"] == "cron":
                 hours = str(job_details["scheduling"]["timing"]["hours"]).zfill(2)
                 minutes = str(job_details["scheduling"]["timing"]["minutes"]).zfill(2)
-                print("Scheduling job " + job_name + " at " + hours + ":" + minutes)
+                logger.info("Scheduling job " + job_name + " at " + hours + ":" + minutes)
 
                 schedule.every().day.at(hours + ":" + minutes).do(run_job,
                                                       job_details=job_details, job_name=job_name)
             elif job_details["scheduling"]["type"] == "interval":
                 minutes = job_details["scheduling"]["timing"].get("minutes", 0)
                 hours = job_details["scheduling"]["timing"].get("hours", 0)
                 total_minutes = minutes + hours*60
-                print("Scheduling job for every " + str(total_minutes) + " minutes")
+                logger.info("Scheduling job for every %d minutes", total_minutes)
 
                 schedule.every(total_minutes).seconds.do(run_job, job_details=job_details,
                                                          job_name=job_name)
 
-if __name__ == '__main__':
-    create_schedule()
+def main():
+    """ The primary interface for this script. """
+    parser = argparse.ArgumentParser(description="External CMR scheduler")
+    parser.add_argument('-t', '--test', action='store_true',
+                        help='Do a test run of RefreashKMSCache and exit.')
+    args = parser.parse_args()
 
+    if args.test:
+        # use these next lines to force a test on a very specific job
+        test_detail = {"target": {"request-type": "POST",
+            "service": "bootstrap",
+            "single-target": True,
+            "endpoint": "caches/refresh/kms"}}
+        run_job(test_detail, "RefreshKMSCache")
+        sys.exit()
+
+    create_schedule()
     while True:
         schedule.run_pending()
         time.sleep(1)
+
+if __name__ == '__main__':
+    main()

job-utilities/local_development/requirements.txt

@@ -1,2 +1,2 @@
-urllib3==1.26.19
+urllib3~=2.5.0
 schedule