Skip to content

Commit 44af191

Browse files
committed
Modified wfbench so that input/output file specifications can be passed
as .json files instead of as command-line JSON strings.
1 parent c054c33 commit 44af191

1 file changed

Lines changed: 43 additions & 12 deletions

File tree

bin/wfbench

Lines changed: 43 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -377,9 +377,11 @@ def get_parser() -> argparse.ArgumentParser:
377377
"Is only approximate since I/O time may make the overall time longer.")
378378
parser.add_argument("--mem", type=float, default=None, help="Maximum memory consumption (in MB).")
379379
parser.add_argument("--output-files", help="Output file names with sizes in bytes as a JSON dictionary "
380-
"(e.g., --output-files {\\\"file1\\\": 1024, \\\"file2\\\": 2048}).")
380+
"(e.g., --output-files \"{\\\"file1\\\": 1024, \\\"file2\\\": 2048}\") OR "
381+
"a path to a .json file that contains that dictionary.")
381382
parser.add_argument("--input-files", help="Input files names as a JSON array "
382-
"(e.g., --input-files [\\\"file3\\\", \\\"file4\\\"]).")
383+
"(e.g., --input-files \"[\\\"file3\\\", \\\"file4\\\"]\") OR "
384+
"a path to a .json file that contains that array.")
383385
parser.add_argument("--verbose", action="store_true", help="Enable all log messages.")
384386
parser.add_argument("--debug", action="store_true", help="Enable debug log messages.")
385387
parser.add_argument("--with-flowcept", action="store_true", default=False, help="Enable Flowcept monitoring.")
@@ -497,11 +499,26 @@ def run(workflow_id, name, with_flowcept, verbose, debug, rundir, path_lock, pat
497499
# is read/written all at once at the beginning/end
498500

499501
# Augment I/O read benchmarks for each input file
500-
cleaned_input = "{}" if input_files is None else re.sub(r'\\+', '', input_files)
501-
try:
502-
input_files = json.loads(cleaned_input)
503-
except json.JSONDecodeError as e:
504-
log_error(f"Failed to decode --input-files JSON string argument: {e}")
502+
503+
if input_files and input_files.endswith(".json"):
504+
try:
505+
with open(input_files, "r") as f:
506+
input_files = json.load(f)
507+
except FileNotFoundError as e:
508+
log_error(f"--input-files JSON file not found: {e}")
509+
sys.exit(1)
510+
except json.JSONDecodeError as e:
511+
log_error(f"Failed to decode JSON in file '{input_files}': {e}")
512+
sys.exit(1)
513+
else:
514+
cleaned_input = "[]" if input_files is None else re.sub(r'\\+', '', input_files)
515+
try:
516+
input_files = json.loads(cleaned_input)
517+
except json.JSONDecodeError as e:
518+
log_error(f"Failed to decode --input-files JSON string argument: {e}")
519+
sys.exit(1)
520+
if not isinstance(input_files, list):
521+
log_error(f"--input-files must be a list JSON object, got {type(input_files).__name__}")
505522
sys.exit(1)
506523

507524
for file_path in input_files:
@@ -520,11 +537,25 @@ def run(workflow_id, name, with_flowcept, verbose, debug, rundir, path_lock, pat
520537
steps[step]["io_read_benchmark"].add_read_operation(file_path, opened_file, num_bytes)
521538

522539
# Augment I/O write benchmarks for each output file
523-
cleaned_output = "{}" if output_files is None else re.sub(r'\\+', '', output_files)
524-
try:
525-
output_files = json.loads(cleaned_output)
526-
except json.JSONDecodeError as e:
527-
log_error(f"Failed to decode --output-files JSON string argument: {e}")
540+
if output_files and output_files.endswith(".json"):
541+
try:
542+
with open(output_files, "r") as f:
543+
output_files = json.load(f)
544+
except FileNotFoundError as e:
545+
log_error(f"--output-files JSON file not found: {e}")
546+
sys.exit(1)
547+
except json.JSONDecodeError as e:
548+
log_error(f"Failed to decode JSON in file '{output_files}': {e}")
549+
sys.exit(1)
550+
else:
551+
cleaned_output = "{}" if output_files is None else re.sub(r'\\+', '', output_files)
552+
try:
553+
output_files = json.loads(cleaned_output)
554+
except json.JSONDecodeError as e:
555+
log_error(f"Failed to decode --output-files JSON string argdument: {e}")
556+
sys.exit(1)
557+
if not isinstance(output_files, dict):
558+
log_error(f"--output-files must be a dict JSON object, got {type(input_files).__name__}")
528559
sys.exit(1)
529560

530561
for file_path, file_size in output_files.items():

0 commit comments

Comments
 (0)