magic-ensemble

#!/usr/bin/env bash
# magic-ensemble: minimal CLI for CCMMF ensemble workflows.
# Usage: ./magic-ensemble <command> [--verbose] --config <path>
# Commands: help | get-demo-data | prepare | prepare-example-1b | prepare-example-2a | prepare-example-3 | run-ensembles

set -euo pipefail

# --- Repo root, manifest, and invocation CWD ---
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$SCRIPT_DIR"
MANIFEST="${REPO_ROOT}/workflow/workflow_manifest.yaml"
INVOCATION_CWD="${INVOCATION_CWD:-$(pwd)}"

# Minimum mikefarah/yq version: the manifest's inputs/outputs/params maps are
# read with 'keys', which is not implemented in early v4 releases (e.g. 4.2.0).
# Bump this if the CLI comes to depend on newer yq features.
MIN_YQ_VERSION="4.53.4"

usage() {
  cat <<'EOF'
Usage: ./magic-ensemble <command> [global options]

Commands:
  help                  Print this usage and help (no scripts run).
  get-demo-data         Fetch demo data from S3 and create run directory (for users without local data).
  prepare               Run canonical preparation steps using workflow/ scripts.
  prepare-example-1b    Run preparation steps using the 1b statewide woody crops example scripts.
  prepare-example-2a    Run preparation steps using the 2a grass example scripts.
  prepare-example-3     Run preparation steps using the 3 row crop example scripts (adds a management-events step).
  run-ensembles         Run step 04 (run model) using existing settings.xml and prepared inputs.

Global options (after command):
  --verbose       Echo each command before running (including apptainer run when use_apptainer is true).
  --config <path> REQUIRED. Path to user YAML config (overridable scalar keys only; fixed paths are in workflow manifest).
                  Config may set use_apptainer: true to run prepare inside Apptainer (pull-if-not-present; apptainer only).

Examples:
  ./magic-ensemble help
  ./magic-ensemble get-demo-data --config my_config.yaml
  ./magic-ensemble prepare --config my_config.yaml --verbose
  ./magic-ensemble prepare-example-2a --config examples/2a_grass/example_user_config.yaml
  ./magic-ensemble prepare-example-1b --config examples/1b_statewide_woody/example_user_config.yaml
  ./magic-ensemble prepare-example-3 --config examples/3_rowcrop/example_user_config.yaml
  ./magic-ensemble run-ensembles --config my_config.yaml
EOF
}

# --- Compare dotted version strings "a.b.c"; returns success (0) if $1 >= $2 ---
version_at_least() {
  local v1="$1" v2="$2"
  local IFS=.
  local -a v1parts=($v1) v2parts=($v2)
  local i a b
  for i in 0 1 2; do
    a="${v1parts[i]:-0}"
    b="${v2parts[i]:-0}"
    if ((10#$a > 10#$b)); then return 0; fi
    if ((10#$a < 10#$b)); then return 1; fi
  done
  return 0
}

# --- Require yq (mikefarah/yq, jq-style) at or above MIN_YQ_VERSION ---
require_yq() {
  if ! command -v yq &>/dev/null; then
    echo "magic-ensemble: yq is required to read YAML. Install mikefarah/yq: https://github.com/mikefarah/yq" >&2
    exit 1
  fi
  if ! yq eval '.' "$MANIFEST" &>/dev/null; then
    echo "magic-ensemble: Could not parse manifest with yq. This CLI requires mikefarah/yq (jq-style). Your 'yq' may be a different implementation." >&2
    exit 1
  fi

  local version_output found_version
  version_output=$(yq --version 2>&1)
  found_version=$(echo "$version_output" | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1)
  if [[ -z "$found_version" ]]; then
    echo "magic-ensemble: Could not determine yq version from '${version_output}'. This CLI requires mikefarah/yq >= ${MIN_YQ_VERSION}." >&2
    exit 1
  fi
  if ! version_at_least "$found_version" "$MIN_YQ_VERSION"; then
    echo "magic-ensemble: yq ${found_version} is too old (found via '$(command -v yq)'). This CLI requires mikefarah/yq >= ${MIN_YQ_VERSION} — older versions don't support 'keys', which the CLI needs to read the workflow manifest's step inputs/outputs/params." >&2
    echo "magic-ensemble: Obtain the newest pecan environment (it ships an up-to-date yq on PATH), or install mikefarah/yq >= ${MIN_YQ_VERSION} directly: https://github.com/mikefarah/yq" >&2
    exit 1
  fi
}

# --- Parse arguments: command first, then global options ---
COMMAND=""
VERBOSE=0
CONFIG_FILE=""
while [[ $# -gt 0 ]]; do
  case "$1" in
    help|get-demo-data|prepare|prepare-example-1b|prepare-example-2a|prepare-example-3|run-ensembles)
      if [[ -z "$COMMAND" ]]; then COMMAND="$1"; shift; continue; fi
      ;;
    --verbose)   VERBOSE=1; shift; continue ;;
    --config)
      if [[ $# -lt 2 ]]; then echo "magic-ensemble: --config requires <path>." >&2; usage >&2; exit 1; fi
      CONFIG_FILE="$2"; shift 2; continue
      ;;
    -*)
      echo "magic-ensemble: Unknown option: $1" >&2; usage >&2; exit 1
      ;;
    *)
      if [[ -z "$COMMAND" ]]; then COMMAND="$1"; shift; continue; fi
      echo "magic-ensemble: Unexpected argument: $1" >&2; usage >&2; exit 1
      ;;
  esac
  shift
done

# Resolve paths passed on the command line relative to CWD (use actual pwd so config is found)
if [[ -n "$CONFIG_FILE" && "$CONFIG_FILE" != /* ]]; then
  CONFIG_FILE="$(pwd)/${CONFIG_FILE}"
fi

# --- Help or no command ---
if [[ -z "$COMMAND" || "$COMMAND" == "help" ]]; then
  usage
  exit 0
fi

if [[ "$COMMAND" != "get-demo-data" && "$COMMAND" != "prepare" && "$COMMAND" != "prepare-example-1b" && "$COMMAND" != "prepare-example-2a" && "$COMMAND" != "prepare-example-3" && "$COMMAND" != "run-ensembles" ]]; then
  echo "magic-ensemble: Unknown command: $COMMAND" >&2
  usage >&2
  exit 1
fi

require_yq
if [[ ! -f "$MANIFEST" ]]; then
  echo "magic-ensemble: Workflow manifest not found: $MANIFEST" >&2
  exit 1
fi

if [[ -z "$CONFIG_FILE" ]]; then
  echo "magic-ensemble: --config <path> is required for command '$COMMAND'." >&2
  usage >&2
  exit 1
fi

if [[ ! -f "$CONFIG_FILE" ]]; then
  echo "magic-ensemble: Config file not found: $CONFIG_FILE" >&2
  exit 1
fi

# --- Load effective config: manifest + optional user overrides.
# key is the full yq expression to evaluate (e.g. ".run_dir", ".paths.foo") —
# get_val does not prepend anything, so what's passed is what's evaluated. ---
get_val() {
  local key="$1"
  local fallback="$2"
  local v
  if [[ -n "$CONFIG_FILE" && -f "$CONFIG_FILE" ]]; then
    v=$(yq eval "$key" "$CONFIG_FILE" 2>/dev/null)
    if [[ -n "$v" && "$v" != "null" ]]; then
      echo "$v"
      return
    fi
  fi
  if [[ -n "$MANIFEST" && -f "$MANIFEST" ]]; then
    v=$(yq eval "$key" "$MANIFEST" 2>/dev/null)
    if [[ -n "$v" && "$v" != "null" ]]; then
      echo "$v"
      return
    fi
  fi
  echo "$fallback"
}

run_dir=$(get_val ".run_dir" "magic-ensemble-run-directory/")

if [[ -z "$run_dir" || "$run_dir" == "null" ]]; then
  echo "magic-ensemble: run_dir is required in config $CONFIG_FILE" >&2
  exit 1
fi

if [[ "$run_dir" == "/" ]]; then
  echo "magic-ensemble: run_dir cannot be the root directory (/)." >&2
  exit 1
fi

# If run_dir is not absolute, resolve relative to CWD where the CLI was invoked
if [[ "$run_dir" != /* ]]; then
  run_dir="${INVOCATION_CWD}/${run_dir}"
fi
# Normalize run_dir to avoid trailing slashes so joined paths do not contain "//"
run_dir="${run_dir%/}"
mkdir -p "$run_dir"

n_workers=$(get_val ".n_workers" "1")

use_apptainer_raw=$(get_val ".use_apptainer" "false")
# Normalize: true/yes/1 (case-insensitive) => 1; else 0
use_apptainer=0
case "$(echo "$use_apptainer_raw" | tr '[:upper:]' '[:lower:]')" in
  true|yes|1) use_apptainer=1 ;;
esac

pecan_dispatch=$(get_val ".pecan_dispatch" "")
slurm_partition=$(get_val ".slurm_partition" "")
aws_profile=$(get_val ".aws_profile" "magic")

# Resolve manifest paths relative to run_dir.
# Effective path = run_dir / manifest_path so R (CWD=REPO_ROOT) sees the correct file.
resolve_path() {
  local key="$1" rel
  rel=$(get_val ".paths.${key}" "" 2>/dev/null)
  if [[ -z "$rel" || "$rel" == "null" ]]; then
    echo "magic-ensemble: no path defined for key '${key}' in manifest" >&2
    exit 1
  fi
  echo "${run_dir}/${rel}"
}



# landtrendr_raw_files is comma-separated; resolve each segment (manifest only for now)
landtrendr_raw_files=""
while IFS= read -r segment; do
  segment=$(echo "$segment" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')
  [[ -z "$segment" ]] && continue
  [[ -n "$landtrendr_raw_files" ]] && landtrendr_raw_files="${landtrendr_raw_files},"
  landtrendr_raw_files="${landtrendr_raw_files}${run_dir}/${segment}"
done < <(yq eval '.paths.landtrendr_raw_files' "$MANIFEST" | tr ',' '\n')

# --- Pre-execution: AWS S3 tools check ---
check_aws() {
  if ! command -v aws &>/dev/null; then
    echo "magic-ensemble: AWS CLI (aws) not found on PATH; required for S3 access." >&2
    exit 1
  fi
}

# --- Apptainer: ensure apptainer is available (module load then PATH); apptainer only, no singularity ---
ensure_apptainer_available() {
  if command -v apptainer &>/dev/null; then
    return 0
  fi
  if command -v module &>/dev/null; then
    if module load apptainer 2>/dev/null; then
      if command -v apptainer &>/dev/null; then
        return 0
      fi
    fi
  fi
  echo "magic-ensemble: use_apptainer is true but apptainer is not available. Run 'module load apptainer' or ensure apptainer is on PATH. (Singularity is not supported.)" >&2
  exit 1
}

# --- Apptainer: resolve SIF path from manifest (in run_dir); pull from remote if not present (no user override of remote) ---
ensure_sif_present() {
  local sif_name sif_path remote_base container_name tag uri
  sif_name=$(yq eval '.apptainer.sif' "$MANIFEST")
  sif_path="${run_dir}/${sif_name}"
  if [[ -f "$sif_path" ]]; then
    APPTAINER_SIF="$sif_path"
    return 0
  fi
  ensure_apptainer_available
  remote_base=$(yq eval '.apptainer.remote.url' "$MANIFEST")
  remote_base="${remote_base%/}"
  container_name=$(yq eval '.apptainer.container.name' "$MANIFEST")
  tag=$(yq eval '.apptainer.tag' "$MANIFEST")
  uri="${remote_base}/${container_name}:${tag}"
  echo "magic-ensemble: SIF not found at $sif_path; pulling from $uri" >&2
  mkdir -p "$run_dir"
  if ! apptainer pull "$sif_path" "$uri"; then
    echo "magic-ensemble: Failed to pull container to $sif_path" >&2
    exit 1
  fi
  APPTAINER_SIF="$sif_path"
}

# --- Minimal log line, styled loosely after PEcAn.logger output ---
log_info() {
  printf '%s %s : %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "INFO" "$1"
}

# --- Prepare summary: SIF/sipnet/R versions, printed after a successful prepare ---
report_prepare_summary() {
  local sipnet_version r_version sif_tag
  r_version=$(Rscript --version 2>&1 | head -1)

  if [[ $use_apptainer -eq 1 ]]; then
    sif_tag=$(yq eval '.apptainer.tag' "$MANIFEST")
    sipnet_version=$(apptainer run "$APPTAINER_SIF" sipnet.git -v 2>&1 | head -1)
    log_info "SIF: ${APPTAINER_SIF} (tag ${sif_tag})"
  else
    sipnet_version=$("${run_dir}/sipnet.git" -v 2>&1 | head -1)
  fi

  log_info "sipnet version: ${sipnet_version}"
  log_info "R version: ${r_version}"
}

# --- Populate STEPS array for current command (from manifest) ---
get_steps_array() {
  STEPS=()
  STEP_NAMES=()
  local script name
  while IFS= read -r script; do
    [[ -z "$script" || "$script" == "null" ]] && continue
    STEPS+=("$script")
  done < <(yq eval ".steps.${COMMAND}[].script" "$MANIFEST" 2>/dev/null)
  while IFS= read -r name; do
    [[ -z "$name" || "$name" == "null" ]] && continue
    STEP_NAMES+=("$name")
  done < <(yq eval ".steps.${COMMAND}[].name" "$MANIFEST" 2>/dev/null)
}

# --- R library check for step at index i (reads r_libraries from manifest step; skip if empty or .sh) ---
check_r_libs_for_step() {
  local i="$1"
  local script="${STEPS[i]}"
  [[ "$script" == *.sh ]] && return 0
  local lib
  while IFS= read -r lib; do
    [[ -z "$lib" || "$lib" == "null" ]] && continue
    if ! (cd "$REPO_ROOT" && Rscript -e "library(\"$lib\")") 2>/dev/null; then
      echo "magic-ensemble: R library check failed: library(\"$lib\") not available. Install it or activate the correct environment." >&2
      exit 1
    fi
  done < <(yq eval '.steps["'"$COMMAND"'"] | .['"$i"'].r_libraries | .[]?' "$MANIFEST" 2>/dev/null || true)
}

# --- R library check for step at index i inside Apptainer (APPTAINER_SIF must be set) ---
check_r_libs_for_step_in_apptainer() {
  local i="$1"
  local script="${STEPS[i]}"
  [[ "$script" == *.sh ]] && return 0
  local lib
  while IFS= read -r lib; do
    [[ -z "$lib" || "$lib" == "null" ]] && continue
    if ! apptainer run --bind "$REPO_ROOT:$REPO_ROOT" --bind "$run_dir:$run_dir" --pwd "$REPO_ROOT" "$APPTAINER_SIF" Rscript -e "library(\"$lib\")" 2>/dev/null; then
      echo "magic-ensemble: R library check failed inside container: library(\"$lib\") not available in image $APPTAINER_SIF" >&2
      exit 1
    fi
  done < <(yq eval '.steps["'"$COMMAND"'"] | .['"$i"'].r_libraries | .[]?' "$MANIFEST" 2>/dev/null || true)
}

# --- Run R script with args. Optional leading args: --apptainer, --cwd DIR (default DIR = REPO_ROOT). ---
run_script() {
  local use_apptainer=0
  local script_cwd="$REPO_ROOT"
  while [[ $# -gt 0 ]]; do
    case "${1:-}" in
      '') shift ;;
      --apptainer) use_apptainer=1; shift ;;
      --cwd)
        if [[ $# -lt 2 ]]; then
          echo "magic-ensemble: run_script --cwd requires DIR." >&2
          exit 1
        fi
        script_cwd="$2"
        shift 2
        ;;
      *) break ;;
    esac
  done
  local script="$1"
  shift
  local script_path="${REPO_ROOT}/${script}"
  if [[ ! -f "$script_path" ]]; then
    echo "magic-ensemble: Script not found: $script_path" >&2
    exit 1
  fi
  if [[ $use_apptainer -eq 1 ]]; then
    # APPTAINER_SIF and run_dir must be set (ensure_sif_present and run_dir resolved earlier).
    echo "magic-ensemble: Rscript (inside apptainer: $APPTAINER_SIF)"
    if [[ $VERBOSE -eq 1 ]]; then
      echo "apptainer run --bind \"$REPO_ROOT:$REPO_ROOT\" --bind \"$run_dir:$run_dir\" --cwd \"$script_cwd\" \"$APPTAINER_SIF\" Rscript \"$script_path\" $*" >&2
    fi
    apptainer run --bind "$REPO_ROOT:$REPO_ROOT" --bind "$run_dir:$run_dir" --cwd "$script_cwd" "$APPTAINER_SIF" Rscript "$script_path" "$@"
  else
    echo "magic-ensemble: Rscript: $(command -v Rscript)"
    if [[ $VERBOSE -eq 1 ]]; then
      echo "(cd \"$script_cwd\" && Rscript \"$script_path\" $*)" >&2
    fi
    (cd "$script_cwd" && Rscript "$script_path" "$@")
  fi
}

# --- Run shell script; CWD = REPO_ROOT. Step scripts receive documented CLI arguments. ---
run_shell_script() {
  local script="$1"
  local step_index="${2:-0}"
  local script_path="${REPO_ROOT}/${script}"
  if [[ ! -f "$script_path" ]]; then
    echo "magic-ensemble: Script not found: $script_path" >&2
    exit 1
  fi
  local script_basename="${script##*/}"
  if [[ "$script_basename" == "00_fetch_s3_and_prepare_run_dir.sh" || "$script_basename" == "00_stage_external_inputs.sh" ]]; then
    # Step 00 helpers: documented arguments (see each script's --help).
    local args=(--repo-root "$REPO_ROOT" --manifest "$MANIFEST")
    if [[ "$script_basename" == "00_fetch_s3_and_prepare_run_dir.sh" ]]; then
      args+=(--command "$COMMAND" --step-index "$step_index" --aws-profile "$aws_profile")
    fi
    if [[ -n "$CONFIG_FILE" && -f "$CONFIG_FILE" ]]; then
      args+=(--config "$CONFIG_FILE" --invocation-cwd "$INVOCATION_CWD")
    else
      args+=(--run-dir "$run_dir" --invocation-cwd "$INVOCATION_CWD")
    fi
    if [[ $VERBOSE -eq 1 ]]; then
      echo "bash $script_path ${args[*]}" >&2
    fi
    (cd "$REPO_ROOT" && bash "$script_path" "${args[@]}")
  else
    # Other shell steps: pass args if/when they are added; no env vars
    if [[ $VERBOSE -eq 1 ]]; then
      echo "bash $script_path" >&2
    fi
    (cd "$REPO_ROOT" && bash "$script_path")
  fi
}

# --- Ensure SIPNET binary is present in run_dir (host-native runs only) ---
ensure_sipnet_binary() {
  local src_dir bin_dir revision
  src_dir="${REPO_ROOT}/$(get_val '.sipnet_install.src_dir' '')"
  bin_dir="${REPO_ROOT}/$(get_val '.sipnet_install.bin_dir' '')"
  revision=$(get_val '.sipnet_install.revision' "")
  [[ "$revision" == "null" ]] && revision=""

  echo "magic-ensemble: ensuring SIPNET binary in $run_dir"
  local expected_bin="${run_dir}/sipnet.git"
  local rev_substr="${revision//v}" #NB allows hashes too (sipnet -v output includes first 10 digits)
  if [[ ! -x "$expected_bin" ]] || [[ $("$expected_bin" -v 2>/dev/null) != *"$rev_substr"* ]]; then
    bash "${REPO_ROOT}/tools/install_sipnet.sh" \
      "$src_dir" "$bin_dir" "$expected_bin" "$revision"
  fi
}

# --- Create the directory an output key needs before its script runs.
# Directory-valued keys (manifest value has no '.', e.g. "cache") are created
# directly; file-valued keys (e.g. "output_prepare/x.csv") have their parent
# directory created instead. ---
ensure_output_dir() {
  local key="$1" resolved="$2" rel
  rel=$(yq eval ".paths.${key}" "$MANIFEST" 2>/dev/null)
  if [[ "$rel" == *.* ]]; then
    mkdir -p "$(dirname "$resolved")"
  else
    mkdir -p "$resolved"
  fi
}

# --- Validate pecan_dispatch value against manifest options ---
validate_pecan_dispatch() {
  if ! yq eval ".pecan_dispatch | has(\"$pecan_dispatch\")" "$MANIFEST" | grep -q '^true$'; then
    echo "magic-ensemble: Unknown pecan_dispatch value '$pecan_dispatch'. Valid options:" >&2
    yq eval '.pecan_dispatch | keys | .[]' "$MANIFEST" >&2
    exit 1
  fi
}

# --- Patch a named XML block in the staged template.xml from manifest values ---
# Usage: patch_xml_block <xml_tag> <yq_path>
#
# Reads the XML block from the manifest at <yq_path>.
# If use_apptainer=1, runs @SIF@ substitution (no-op when @SIF@ is absent) and
# overrides @SIPNET_BINARY@ to the correct path for the container
patch_xml_block() {
  local xml_tag="$1" yq_path="$2"

  if ! command -v python3 &>/dev/null; then
    echo "magic-ensemble: python3 is required to patch <${xml_tag}> in template.xml." >&2
    exit 1
  fi

  local template_path="${run_dir}/$(yq eval '.paths.template_file' "$MANIFEST")"
  if [[ ! -f "$template_path" ]]; then
    echo "magic-ensemble: staged template.xml not found at $template_path" >&2
    exit 1
  fi

  local sif_name=""
  local sipnet_binary="${run_dir}/sipnet.git"
  if [[ $use_apptainer -eq 1 ]]; then
    sif_name=$(yq eval '.apptainer.sif' "$MANIFEST")
    if [[ -n "$sif_name" ]]; then
      sif_name="apptainer run ./${sif_name}"
    fi
    sipnet_binary=$(yq eval '.apptainer.sipnet_binary' "$MANIFEST")
  fi

  if [[ $VERBOSE -eq 1 ]]; then
    echo "magic-ensemble: patching <${xml_tag}> block in $template_path (yq_path=${yq_path})" >&2
  fi

  local xml_block partition_flag=""
  if [[ -n "$slurm_partition" && "$slurm_partition" != "null" ]]; then
    partition_flag="-p ${slurm_partition}"
  fi
  xml_block=$(yq eval "$yq_path" "$MANIFEST" \
    | sed "s|@SIF@|${sif_name}|g" \
    | sed "s|@PARTITION@|${partition_flag}|g" \
    | sed "s|@SIPNET_BINARY@|${sipnet_binary}|g" \
    | sed "s|@NCPUS@|${n_workers}|g")

  python3 "${REPO_ROOT}/tools/patch_xml.py" "$template_path" "$xml_tag" "$xml_block" --block
}


# --- Build the full argument list (STEP_ARGS) for one named step of the
# current command, entirely from the manifest's inputs/outputs/params for
# that step. This is the only place that turns manifest declarations into
# --flag values; no script-specific code lives here or in the command
# handlers below.
#
# inputs/outputs/params are dicts of {cli_flag: value_key}: the dict key is
# the literal flag name passed on the command line, the dict value names
# where the value actually comes from (a 'paths' entry for inputs/outputs,
# a get_val-resolvable key for params). Flag name and value key are often
# identical, but don't have to be — this is what lets a script's flag name
# differ from a path's canonical identity without duplicating manifest
# paths just to satisfy it. ---
build_step_args() {
  local name="$1"
  STEP_ARGS=()

  # Every command substitution below is followed by "|| exit 1" explicitly —
  # do not rely on set -e propagating through "arr+=(\"$(cmd)\")"; that
  # behavior is not reliable enough across bash versions to trust silently.
  #
  # Path-valued inputs
  local flag value_key resolved param_value
  while IFS= read -r flag; do
    [[ -z "$flag" || "$flag" == "null" ]] && continue
    value_key=$(yq eval ".steps.${COMMAND}[] | select(.name == \"${name}\") | .inputs[\"${flag}\"]" "$MANIFEST" 2>/dev/null)
    resolved=$(resolve_path "$value_key") || exit 1
    STEP_ARGS+=("--${flag}" "$resolved")
  done < <(yq eval ".steps.${COMMAND}[] | select(.name == \"${name}\") | .inputs | keys | .[]?" "$MANIFEST" 2>/dev/null)

# path-valued outputs
  while IFS= read -r flag; do
    [[ -z "$flag" || "$flag" == "null" ]] && continue
    value_key=$(yq eval ".steps.${COMMAND}[] | select(.name == \"${name}\") | .outputs[\"${flag}\"]" "$MANIFEST" 2>/dev/null)
    resolved=$(resolve_path "$value_key") || exit 1
    ensure_output_dir "$value_key" "$resolved"
    # a flag can appear in both inputs and outputs (read-modify-write dirs
    # like data_dir); don't pass the same flag twice.
    if [[ " ${STEP_ARGS[*]} " != *" --${flag} "* ]]; then
      STEP_ARGS+=("--${flag}" "$resolved")
    fi
  done < <(yq eval ".steps.${COMMAND}[] | select(.name == \"${name}\") | .outputs | keys | .[]?" "$MANIFEST" 2>/dev/null)

# scalar parameter values
  while IFS= read -r flag; do
    [[ -z "$flag" || "$flag" == "null" ]] && continue
    value_key=$(yq eval ".steps.${COMMAND}[] | select(.name == \"${name}\") | .params[\"${flag}\"]" "$MANIFEST" 2>/dev/null)
    # TODO magic-downscaling uses a `resolve_param` fn for more elaborate validation
    # of lists and scalars here -- consider copying?
    param_value=$(get_val ".${value_key}" "") || exit 1
    STEP_ARGS+=("--${flag}" "$param_value")
  done < <(yq eval ".steps.${COMMAND}[] | select(.name == \"${name}\") | .params | keys | .[]?" "$MANIFEST" 2>/dev/null)
}

# Generic handler for every manifest-driven command:
# step list, script paths, and arguments all come from the manifest.
# Adding, removing, or reordering a step only requires editing manifest.yaml.
run_manifest_command() {
  local logfile="$run_dir"/"$COMMAND"_$(date +'%Y%m%d_%H%M%S').log
  { # block for output redirection
    get_steps_array
    validate_pecan_dispatch
    local apptainer_arg=""
    if [[ $use_apptainer -eq 1 ]]; then
      ensure_apptainer_available
      ensure_sif_present
      apptainer_arg="--apptainer"
      for i in "${!STEPS[@]}"; do
        check_r_libs_for_step_in_apptainer "$i"
      done
    else
      for i in "${!STEPS[@]}"; do
        check_r_libs_for_step "$i"
      done
    fi

    local total="${#STEPS[@]}" i step_num script name
    for i in "${!STEPS[@]}"; do
      script="${STEPS[i]}"
      name="${STEP_NAMES[i]}"
      step_num=$((i + 1))
      echo "magic-ensemble: ${COMMAND} step ${step_num} of ${total}: ${script}"
      build_step_args "$name"
      if [[ "$script" == *.sh ]]; then
        run_shell_script "$script" "$i"
        if [[ "$name" == "stage-inputs" ]]; then
          # TODO give this its own step instead?
          echo "magic-ensemble: patching template.xml with dispatch: $pecan_dispatch"
          patch_xml_block "host" \
            ".pecan_dispatch[\"$pecan_dispatch\"].host_xml"
          patch_xml_block "model/binary" ".sipnet_model.model_binary"
          if [[ $use_apptainer -eq 0 ]]; then
            ensure_sipnet_binary
          fi
        fi
      else
        run_script "$apptainer_arg" "$script" "${STEP_ARGS[@]}"
      fi
      echo "magic-ensemble: ${COMMAND} step ${step_num} completed"
    done
    echo "magic-ensemble: ${COMMAND} finished (${total} steps)"
    echo "magic-ensemble: output logged to ${logfile}."
    if [[ "$COMMAND" == "prepare"* ]]; then
      # TODO make this a step of its own?
      report_prepare_summary
    fi
  } 2>&1 | tee -a ${logfile}
}

# --- Get-demo-data: run steps from manifest (shell script only) ---
run_get_demo_data() {
  get_steps_array
  check_aws
  for i in "${!STEPS[@]}"; do
    check_r_libs_for_step "$i"
    run_shell_script "${STEPS[i]}" "$i"
  done
}


# --- Run-ensembles: run steps from manifest (set-up-runs, run-model); never
# inside Apptainer. When use_apptainer=1, the SIF must be present for
# dispatched jobs (already patched into template.xml via patch_xml_block
# during prepare); these scripts themselves always run on the host so they
# can submit further jobs to Slurm.
run_run_ensembles() {
  local logfile="$run_dir"/"$COMMAND"_$(date +'%Y%m%d_%H%M%S').log
  { # block for output redirection
    get_steps_array

    if [[ $use_apptainer -eq 1 ]]; then
      ensure_apptainer_available
      ensure_sif_present
    fi
    local i script name
    for i in "${!STEPS[@]}"; do
      check_r_libs_for_step "$i"
      script="${STEPS[i]}"
      name="${STEP_NAMES[i]}"
      build_step_args "$name"
      run_script --cwd "$run_dir" "$script" "${STEP_ARGS[@]}"
    done
    echo "magic-ensemble: output logged to ${logfile}."
  } 2>&1 | tee -a ${logfile}
}

# --- Main ---
case "$COMMAND" in
  get-demo-data)        run_get_demo_data ;;
  prepare)              run_manifest_command ;;
  prepare-example-1b)   run_manifest_command ;;
  prepare-example-2a)   run_manifest_command ;;
  prepare-example-3)    run_manifest_command ;;
  run-ensembles)        run_run_ensembles ;;
  *)                    echo "magic-ensemble: Unknown command: $COMMAND" >&2; exit 1 ;;
esac