cli.py

"""Command-line interface for the Fair Code dataset profiler.

    faircode profile data.csv
    faircode profile data.tsv
    faircode profile data.xlsx
    faircode profile data.json
    faircode profile data.parquet
    faircode profile data.csv --json
    faircode profile data.csv --html report.html
    faircode compare train.csv prod.csv
    faircode compare train.csv prod.csv --json
    faircode compare train.csv prod.csv --html report.html
    faircode benchmark
    faircode benchmark --out results/
    faircode benchmark COMPAS/audit.yaml "German Credit Lending/audit.yaml"

Uses only stdlib argparse + pandas (no heavyweight profiling dependency).
Reading .xlsx additionally requires the optional 'openpyxl' extra
(`pip install faircode[excel]`); reading .parquet additionally requires the
optional 'pyarrow' extra (`pip install faircode[parquet]`). The `benchmark`
command additionally requires the optional 'benchmark' extra
(`pip install faircode[benchmark]`: scikit-learn + pyyaml + fairlearn + matplotlib).
"""

from __future__ import annotations

import argparse
import sys

from . import __version__
from .compare import compare
from .detect import VALID_KINDS
from .loaders_extra import get_xlsx_sheet_info, read_table
# _resolve_opts gives the thresholds that were actually in force, defaults
# included, which is what the provenance block has to record. Reaching for the
# private helper follows the existing precedent in compare.py (`from .profiler
# import _r`) and keeps profiler.py - a parity-sensitive file - untouched.
from .profiler import _resolve_opts, parse_reference, profile
from .provenance import build as build_provenance
from .proxy import parse_held_out_specs, proxy_hints
from .report import compare_to_terminal, to_html, compare_to_html, to_json, to_terminal

_MAP_CHOICES = VALID_KINDS + ("ignore",)


def _parse_map(pairs):
    """Parse repeated --map COL=KIND flags into an {column: kind} override dict."""
    overrides = {}
    for pair in pairs or []:
        if "=" not in pair:
            print(f"error: invalid --map '{pair}', expected COL=KIND", file=sys.stderr)
            raise SystemExit(2)
        col, kind = pair.split("=", 1)
        kind = kind.strip().lower()
        if kind not in _MAP_CHOICES:
            print(f"error: invalid --map kind '{kind}' for column '{col.strip()}'; "
                  f"choose from {', '.join(_MAP_CHOICES)}", file=sys.stderr)
            raise SystemExit(2)
        overrides[col.strip()] = kind
    return overrides


def _check_map_columns(overrides, known_columns):
    """Error out on any --map key that isn't an actual column, instead of
    silently no-opping - detect_columns() only applies an override `if col
    in overrides` while iterating real df.columns, so a typo'd column name
    was previously dropped with no feedback at all."""
    unknown = [col for col in overrides if col not in known_columns]
    if unknown:
        print(f"error: --map column(s) not found in the dataset: {', '.join(unknown)}",
              file=sys.stderr)
        raise SystemExit(2)


def _build_held_out(specs, df):
    """Parse repeated --proxy-hints-with PATH=COLUMN flags via proxy.py's
    shared parse_held_out_specs, printing a plain error and raising
    SystemExit(2) on any parse failure, missing column, or row-count
    mismatch - _read_or_exit already does the same for an unreadable path.

    Also prints the same ignored-sheet notice the main dataset path already
    gets for a multi-sheet .xlsx file - a held-out file is a third dataset
    input, and only reading sheet 0 without saying so would otherwise be
    silently inconsistent with every other input path."""
    for spec in specs or []:
        path = spec.partition("=")[0]
        sheet_info = get_xlsx_sheet_info(path)
        if sheet_info is not None:
            sheet_name, ignored_sheets = sheet_info
            if ignored_sheets:
                print(
                    f"{path}: read sheet '{sheet_name}' - {len(ignored_sheets)} "
                    f"other sheet(s) ignored.",
                    file=sys.stderr,
                )
    try:
        return parse_held_out_specs(specs, df, _read_or_exit)
    except ValueError as exc:
        print(f"error: {exc}", file=sys.stderr)
        raise SystemExit(2)


def _read_or_exit(path: str):
    """Read a table, or print a plain error and raise SystemExit(2)."""
    try:
        return read_table(path)
    except FileNotFoundError:
        print(f"error: file not found: {path}", file=sys.stderr)
        raise SystemExit(2)
    except RuntimeError as exc:
        print(f"error: {exc}", file=sys.stderr)
        raise SystemExit(2)
    except Exception as exc:  # noqa: BLE001 - surface any parse failure plainly
        print(f"error: could not read dataset {path}: {exc}", file=sys.stderr)
        raise SystemExit(2)


def main(argv: list[str] | None = None) -> int:
    parser = argparse.ArgumentParser(
        prog="faircode",
        description="Audit a tabular dataset for demographic representation.",
    )
    parser.add_argument("--version", action="version",
                        version=f"faircode {__version__}")
    sub = parser.add_subparsers(dest="command", required=True)

    p = sub.add_parser("profile", help="profile a dataset for demographic imbalance")
    p.add_argument("csv", help="path to the dataset file (.csv, .tsv, .xlsx, .json, or .parquet), "
                               "or - to read CSV/TSV from stdin")
    p.add_argument("--json", action="store_true", help="emit JSON to stdout")
    p.add_argument("--html", metavar="PATH",
                   help="write a standalone HTML report to PATH")
    p.add_argument("--fail-under", type=float, metavar="N",
                   help="exit 1 when the overall representation score is below N")
    p.add_argument("--map", action="append", metavar="COL=KIND",
                   help="force a column's dimension when auto-detection misses it; "
                        "KIND is one of " + ", ".join(_MAP_CHOICES) + " (repeatable)")
    p.add_argument("--cross", metavar="COLA,COLB",
                   help="cross these two columns for the intersectional gap "
                        "(default: the first two detected dimensions)")
    p.add_argument("--reference", metavar="PATH",
                   help="score against a reference baseline dataset (columns: column,group,share)")
    p.add_argument("--proxy-hints", action="store_true",
                   help="flag strongly-associated column pairs via chi-squared (needs scipy)")
    p.add_argument("--proxy-hints-with", action="append", metavar="PATH=COLUMN",
                   help="also test proxy_hints against a column already dropped from "
                        "the dataset; PATH's rows must align 1:1 with the profiled "
                        "dataset (repeatable, needs --proxy-hints)")
    p.add_argument("--min-share", type=float, metavar="F",
                   help="under-representation threshold (default 0.05)")
    p.add_argument("--intersection-floor", type=float, metavar="F",
                   help="near-empty intersection-cell threshold (default 0.01)")
    p.add_argument("--imbalance-flag", type=float, metavar="F",
                   help="imbalance-ratio flag threshold (default 3.0)")
    p.add_argument("--missing-flag", type=float, metavar="F",
                   help="missing-data flag threshold (default 0.05)")
    p.add_argument("--min-group-size", type=int, metavar="N",
                   help="warn when a subgroup has fewer than N rows (default: profiler.MIN_GROUP_SIZE)")
    p.add_argument("--no-provenance", action="store_true",
                   help="omit the provenance block from --json output "
                        "(restores the pre-2.1 export shape exactly)")

    c = sub.add_parser("compare",
                       help="compare two datasets for representation drift")
    c.add_argument("csv_a", help="baseline dataset A (.csv, .tsv, .xlsx, .json, or .parquet), "
                                 "or - to read CSV/TSV from stdin")
    c.add_argument("csv_b", help="current dataset B (.csv, .tsv, .xlsx, .json, or .parquet), "
                                 "or - to read CSV/TSV from stdin")
    c.add_argument("--json", action="store_true", help="emit JSON to stdout")
    c.add_argument("--html", metavar="PATH",
                   help="write a standalone HTML report to PATH")
    c.add_argument("--proxy-hints", action="store_true",
                   help="flag strongly-associated column pairs via chi-squared, "
                        "for both datasets separately (needs scipy)")
    c.add_argument("--map", action="append", metavar="COL=KIND",
                   help="force a column's dimension when auto-detection misses it "
                        "(applied to both datasets); KIND is one of " +
                        ", ".join(_MAP_CHOICES) + " (repeatable)")
    c.add_argument("--min-share", type=float, metavar="F",
                   help="under-representation threshold (default 0.05)")
    c.add_argument("--intersection-floor", type=float, metavar="F",
                   help="near-empty intersection-cell threshold (default 0.01)")
    c.add_argument("--imbalance-flag", type=float, metavar="F",
                   help="imbalance-ratio flag threshold (default 3.0)")
    c.add_argument("--missing-flag", type=float, metavar="F",
                   help="missing-data flag threshold (default 0.05)")
    c.add_argument("--min-group-size", type=int, metavar="N",
                   help="warn when a subgroup has fewer than N rows (default: profiler.MIN_GROUP_SIZE)")
    c.add_argument("--fail-on-drift", action="store_true",
                   help="exit 1 when any dimension shows drift or the overall score drops")
    c.add_argument("--no-provenance", action="store_true",
                   help="omit the provenance block from --json output "
                        "(restores the pre-2.1 export shape exactly)")

    b = sub.add_parser("benchmark",
                       help="run the cross-domain fairness benchmark harness over every audit.yaml")
    b.add_argument("manifests", nargs="*", metavar="audit.yaml",
                   help="explicit manifest paths (default: discover */audit.yaml under --root)")
    b.add_argument("--root", default=".", metavar="PATH",
                   help="directory to search for */audit.yaml (default: current directory)")
    b.add_argument("--out", default="results", metavar="DIR",
                   help="output directory for results_fairness.csv, "
                        "results_performance.csv, summary.csv, and figures/ "
                        "(default: results)")
    b.add_argument("--n-resamples", type=int, default=2000, metavar="N",
                   help="bootstrap resamples per metric (default: 2000)")
    b.add_argument("--n-permutations", type=int, default=2000, metavar="N",
                   help="permutation-test shuffles per metric (default: 2000)")
    b.add_argument("--no-plots", action="store_true",
                   help="skip rendering figures/*.png (no matplotlib needed)")

    args = parser.parse_args(argv)

    if args.command == "profile":
        if args.proxy_hints_with and not args.proxy_hints:
            print("error: --proxy-hints-with needs --proxy-hints", file=sys.stderr)
            return 2

        df = _read_or_exit(args.csv)

        sheet_info = get_xlsx_sheet_info(args.csv)
        if sheet_info is not None:
            sheet_name, ignored_sheets = sheet_info
            if ignored_sheets:
                print(
                    f"Read sheet '{sheet_name}' - {len(ignored_sheets)} "
                    f"other sheet(s) ignored.",
                    file=sys.stderr,
                )

        opts = {
            "min_share": args.min_share,
            "intersection_floor": args.intersection_floor,
            "imbalance_flag": args.imbalance_flag,
            "missing_flag": args.missing_flag,
            "min_group_size": args.min_group_size,
        }
        if args.cross:
            parts = [c.strip() for c in args.cross.split(",")]
            if len(parts) != 2 or not all(parts):
                print("error: --cross expects two column names: COLA,COLB",
                      file=sys.stderr)
                return 2
            if parts[0] == parts[1]:
                print("error: --cross needs two different columns, got "
                      f"'{parts[0]}' twice", file=sys.stderr)
                return 2
            opts["cross"] = parts
        if args.reference:
            try:
                opts["reference"] = parse_reference(_read_or_exit(args.reference))
            except ValueError as exc:
                print(f"error: {exc}", file=sys.stderr)
                return 2

        overrides = _parse_map(args.map)
        _check_map_columns(overrides, df.columns)
        try:
            result = profile(df, overrides, opts)
        except ValueError as exc:
            print(f"error: {exc}", file=sys.stderr)
            return 2

        if args.proxy_hints or args.proxy_hints_with:
            held_out = _build_held_out(args.proxy_hints_with, df)
            try:
                result["proxy_hints"] = proxy_hints(df, result["dimensions"], held_out=held_out)
            except RuntimeError as exc:
                print(f"error: {exc}", file=sys.stderr)
                return 2

        if args.html:
            html_content = to_html(result)
            try:
                with open(args.html, "w", encoding="utf-8") as fh:
                    fh.write(html_content)
            except OSError as exc:
                print(f"error: could not write HTML report to {args.html}: {exc}",
                      file=sys.stderr)
                return 2
            print(f"HTML report written to {args.html}", file=sys.stderr)

        if args.json:
            provenance = None
            if not args.no_provenance:
                digests = [("dataset_hash", args.csv)]
                if args.reference:
                    digests.append(("reference_hash", args.reference))
                provenance = build_provenance(digests, _resolve_opts(opts), overrides)
            print(to_json(result, provenance=provenance))
        else:
            print(to_terminal(result))
        if args.fail_under is not None and result["overall_score"] is None:
            print(
                "error: cannot apply --fail-under: no demographic columns detected",
                file=sys.stderr,
            )
            return 2
        if args.fail_under is not None and result["overall_score"] < args.fail_under:
            print(
                f"error: representation score {result['overall_score']}/100 is below "
                f"--fail-under {args.fail_under:g}",
                file=sys.stderr,
            )
            return 1
        return 0

    if args.command == "compare":
        if args.csv_a == "-" and args.csv_b == "-":
            print("error: --compare can't read both datasets from stdin "
                  "(a stream can only be read once)", file=sys.stderr)
            return 2
        overrides = _parse_map(args.map)
        opts = {
            "min_share": args.min_share,
            "intersection_floor": args.intersection_floor,
            "imbalance_flag": args.imbalance_flag,
            "missing_flag": args.missing_flag,
            "min_group_size": args.min_group_size,
        }
        df_a = _read_or_exit(args.csv_a)
        df_b = _read_or_exit(args.csv_b)
        _check_map_columns(overrides, set(df_a.columns) | set(df_b.columns))

        for path in (args.csv_a, args.csv_b):
            sheet_info = get_xlsx_sheet_info(path)
            if sheet_info is not None:
                sheet_name, ignored_sheets = sheet_info
                if ignored_sheets:
                    print(
                        f"{path}: read sheet '{sheet_name}' - {len(ignored_sheets)} "
                        f"other sheet(s) ignored.",
                        file=sys.stderr,
                    )

        profile_a = profile(df_a, overrides, opts)
        profile_b = profile(df_b, overrides, opts)
        result = compare(profile_a, profile_b, name_a=args.csv_a, name_b=args.csv_b)

        if args.proxy_hints:
            try:
                result["proxy_hints_a"] = proxy_hints(df_a, profile_a["dimensions"])
                result["proxy_hints_b"] = proxy_hints(df_b, profile_b["dimensions"])
            except RuntimeError as exc:
                print(f"error: {exc}", file=sys.stderr)
                return 2

        if args.html:
            html_content = compare_to_html(result)
            try:
                with open(args.html, "w", encoding="utf-8") as fh:
                    fh.write(html_content)
            except OSError as exc:
                print(f"error: could not write HTML report to {args.html}: {exc}",
                      file=sys.stderr)
                return 2
            print(f"HTML report written to {args.html}", file=sys.stderr)
        if args.json:
            provenance = None
            if not args.no_provenance:
                provenance = build_provenance(
                    [("dataset_hash_a", args.csv_a), ("dataset_hash_b", args.csv_b)],
                    _resolve_opts(opts), overrides)
            print(to_json(result, provenance=provenance))
        else:
            print(compare_to_terminal(result))
        if args.fail_on_drift and result["drift_detected"]:
            print(
                f"error: representation drift detected ({len(result['flags'])} flag(s)) "
                f"with --fail-on-drift set",
                file=sys.stderr,
            )
            return 1
        return 0

    if args.command == "benchmark":
        try:
            from .benchmark import run_benchmark, write_report
        except ImportError as exc:
            print(f"error: the benchmark command needs scikit-learn and pyyaml "
                  f"(pip install faircode[benchmark]): {exc}", file=sys.stderr)
            return 2

        overridden = []
        if args.n_resamples != 2000:
            overridden.append(f"--n-resamples={args.n_resamples}")
        if args.n_permutations != 2000:
            overridden.append(f"--n-permutations={args.n_permutations}")

        if overridden:
            print(
                "warning: "
                + ", ".join(overridden)
                + " differs from the frozen paper-run default (2000); "
                "output will not match the frozen paper reference.",
                file=sys.stderr,
            )

        try:
            fairness_df, performance_df = run_benchmark(
                root=args.root, audits=args.manifests or None,
                n_resamples=args.n_resamples, n_permutations=args.n_permutations,
            )
        except ValueError as exc:
            print(f"error: {exc}", file=sys.stderr)
            return 2
        if fairness_df.empty:
            print(f"error: no audit.yaml manifests found under {args.root}", file=sys.stderr)
            return 2

        try:
            write_report(fairness_df, performance_df, args.out, make_plots=not args.no_plots)
        except ImportError as exc:
            print(f"error: writing benchmark plots needs matplotlib "
                  f"(pip install faircode[benchmark]): {exc}", file=sys.stderr)
            return 2
        n_audits = fairness_df["audit"].nunique()
        print(f"Ran {n_audits} audit(s), wrote {len(fairness_df)} fairness rows and "
              f"{len(performance_df)} performance rows to {args.out}/", file=sys.stderr)
        return 0

    parser.print_help()
    return 1


if __name__ == "__main__":
    raise SystemExit(main())