fix: Update ini import (#212)

update ini import to deal with custom variables in the ini file.

---------

Co-authored-by: Eric Novotny <nov00002@umn.edu>

Author: Enovotny

cwmscli/usgs/__init__.py

@@ -105,15 +105,23 @@ def getusgs_ratings(office, days_back, api_root, api_key, api_key_loc, rating_su
     type=str,
     help="filename of ratings ini file to be processed",
 )
+@click.option(
+    "--dry-run",
+    is_flag=True,
+    default=False,
+    help="Preview changes without updating the database",
+)
 @api_root_option
 @api_key_option
 @api_key_loc_option
 @requires(reqs.cwms, reqs.requests)
-def ratingsinifileimport(filename, api_root, api_key, api_key_loc):
+def ratingsinifileimport(filename, dry_run, api_root, api_key, api_key_loc):
     from cwmscli.usgs.rating_ini_file_import import rating_ini_file_import
 
     api_key = get_api_key(api_key, api_key_loc)
-    rating_ini_file_import(api_root=api_root, api_key=api_key, ini_filename=filename)
+    rating_ini_file_import(
+        api_root=api_root, api_key=api_key, ini_filename=filename, dry_run=dry_run
+    )
 
 
 @usgs_group.command("measurements", help="Store USGS measurements into CWMS database")

cwmscli/usgs/rating_ini_file_import.py

@@ -11,8 +11,12 @@
 }
 
 
-def rating_ini_file_import(api_root, api_key, ini_filename):
-    init_cwms_session(cwms, api_root=api_root, api_key="apikey " + api_key)
+def rating_ini_file_import(api_root, api_key, ini_filename, dry_run=False):
+    if dry_run:
+        logging.info("DRY RUN MODE - no changes will be made")
+        init_cwms_session(cwms, api_root=api_root)
+    else:
+        init_cwms_session(cwms, api_root=api_root, api_key="apikey " + api_key)
 
     logging.info(f"CDA connection: {api_root}")
     logging.info(f"Opening ini file: {ini_filename}")
@@ -21,7 +25,6 @@ def rating_ini_file_import(api_root, api_key, ini_filename):
     ini_file.close()
 
     params = {}
-    keywords = ["cwms_office", "db_base", "db_exsa", "db_corr", "localid"]
     rating_errors = []
     for i in range(len(lines)):
         line = lines[i][:-1].strip()
@@ -33,26 +36,41 @@ def rating_ini_file_import(api_root, api_key, ini_filename):
             continue
         if "=" in line:
             fields = line.split("=")
-            if fields[0] in keywords:
-                if fields[0] == "cwms_office":
-                    fields[1] = fields[1].upper()
-                params[fields[0]] = fields[1]
+            key = fields[0].strip().lower()
+            value = fields[1].strip()
+            if key == "cwms_office":
+                value = value.upper()
+            params[key] = value
         else:
             fields = parse_ini_line(line)
             if fields[0] in rating_types.keys():
-                rating_db_type = rating_types[fields[0]]["db_type"]
-                if f"$(${rating_db_type})" in fields:
-                    rating_spec = params[rating_db_type].replace(
-                        "\$localid", params["localid"]
-                    )
+                # Find the database reference in the fields (e.g., $($db_tail), $($db_exsa), etc.)
+                db_key = None
+                for field in fields[1:]:
+                    if field.startswith("$(") and field.endswith(")"):
+                        # Extract the key name from $(...), e.g., "db_exsa" from "$($db_exsa)"
+                        potential_key = field[2:-1].lstrip("$")
+                        if potential_key in params:
+                            db_key = potential_key
+                            break
+
+                if db_key:
+                    rating_spec = params[db_key]
+                    # Substitute any custom parameters found in rating_spec
+                    for param_key, param_value in params.items():
+                        placeholder = f"\\${param_key}"
+                        if placeholder in rating_spec:
+                            rating_spec = rating_spec.replace(placeholder, param_value)
                     logging.info(f"Updating rating specification: {rating_spec}")
                     try:
                         update_rating_spec(
                             rating_spec,
-                            params["cwms_office"],
+                            params.get("cwms_office"),
                             rating_types[fields[0]]["db_disc"],
+                            dry_run=dry_run,
                         )
-                        logging.info("SUCCESS: rating specification changes stored")
+                        if not dry_run:
+                            logging.info("SUCCESS: rating specification changes stored")
                     except:
                         logging.error(
                             "ERROR: rating specificataion could not be update"
@@ -109,7 +127,7 @@ def parse_ini_line(line):
     return fields
 
 
-def update_rating_spec(rating_id, office_id, db_disc):
+def update_rating_spec(rating_id, office_id, db_disc, dry_run=False):
     rating_spec = cwms.get_rating_spec(rating_id=rating_id, office_id=office_id)
     data = rating_spec.df
     data = data.drop("effective-dates", axis=1)
@@ -127,4 +145,8 @@ def update_rating_spec(rating_id, office_id, db_disc):
     disc = data.loc[0, "description"]
     logging.info(f"Saving specification discription as: {disc}")
     data_xml = cwms.rating_spec_df_to_xml(data)
-    cwms.store_rating_spec(data=data_xml, fail_if_exists=False)
+    if dry_run:
+        logging.info("DRY RUN: Would store rating specification with XML:")
+        logging.info(data_xml)
+    else:
+        cwms.store_rating_spec(data=data_xml, fail_if_exists=False)