Scoring-Bias-in-LLM-as-a-Judge/Makefile at main · ssamba1/Scoring-Bias-in-LLM-as-a-Judge · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
.PHONY: help install test lint figures paper archive ci setup clean reproduce-all pre-commit \
        install-package check-credentials health-check \
        validate docs integrity verify-clean verify-like-ci arxiv-package

help:  # Show available targets
	@echo "╔══════════════════════════════════════════════════════════════╗"
	@echo "║   Scoring Bias  Makefile Help                              ║"
	@echo "╚══════════════════════════════════════════════════════════════╝"
	@echo ""
	@echo "── Development ────────────────────────────────────────────────"
	@printf "  \033[36m%-22s\033[0m %s\n" "make setup" "Set up development environment"
	@printf "  \033[36m%-22s\033[0m %s\n" "make install" "Install Python dependencies + pre-commit"
	@printf "  \033[36m%-22s\033[0m %s\n" "make install-package" "Install scoring-bias package in dev mode"
	@printf "  \033[36m%-22s\033[0m %s\n" "make pre-commit" "Run pre-commit on all files"
	@echo ""
	@echo "── Testing & Quality ──────────────────────────────────────────"
	@printf "  \033[36m%-22s\033[0m %s\n" "make test" "Run all unit tests with pytest"
	@printf "  \033[36m%-22s\033[0m %s\n" "make test-cov" "Run tests with coverage report"
	@printf "  \033[36m%-22s\033[0m %s\n" "make lint" "Run flake8 + black (check mode)"
	@printf "  \033[36m%-22s\033[0m %s\n" "make ci" "Run test + lint (CI pipeline)"
	@echo ""
	@echo "── Paper & Figures ────────────────────────────────────────────"
	@printf "  \033[36m%-22s\033[0m %s\n" "make paper" "Compile paper PDF from LaTeX"
	@printf "  \033[36m%-22s\033[0m %s\n" "make figures" "Regenerate all publication figures"
	@printf "  \033[36m%-22s\033[0m %s\n" "make archive" "Generate arXiv submission package"
	@echo ""
	@echo "── Data & Validation ──────────────────────────────────────────"
	@printf "  \033[36m%-22s\033[0m %s\n" "make validate" "Run data validation pipeline"
	@echo ""
	@echo "── Infrastructure ─────────────────────────────────────────────"
	@printf "  \033[36m%-22s\033[0m %s\n" "make docs" "Build project documentation"
	@printf "  \033[36m%-22s\033[0m %s\n" "make clean" "Remove all build artifacts and caches"
	@printf "  \033[36m%-22s\033[0m %s\n" "make check-credentials" "Scan for accidentally committed credentials"
	@printf "  \033[36m%-22s\033[0m %s\n" "make health-check" "Verify project integrity"
	@printf "  \033[36m%-22s\033[0m %s\n" "make verify-like-ci" "Run every CI job locally"
	@echo ""
	@echo "── Pipeline ───────────────────────────────────────────────────"
	@printf "  \033[36m%-22s\033[0m %s\n" "make reproduce-all" "Full end-to-end reproduction: setup → test → paper → archive"

setup: install install-package pre-commit  # Set up development environment

install:  # Install Python dependencies
	pip install -U pip
	pip install -r requirements.txt
	pip install pre-commit pytest pytest-cov
	pre-commit install

install-package:  # Install the scoring-bias package in dev mode
	pip install -e ".[dev,api,dashboard,notebook]"

pre-commit:  # Run pre-commit checks on all files
	pre-commit run --all-files

test:  # Run all unit tests with pytest
	python -m pytest tests/ -v --tb=short

test-cov:  # Run tests with coverage report
	python -m pytest tests/ -v --tb=short --cov=src --cov-report=term-missing

# flake8 reads its settings from setup.cfg, including why mutation_check.py is
# exempt from the line-length rule.
lint:  # Run code quality checks (flake8)
	pip install flake8 -q
	flake8 tests/ mutation_check.py verify_like_ci.py scan_secrets.py \
		--count --statistics

# black is NOT part of the gate, and saying so is the point. It has never been
# run here: 72 of the 88 files it covers would be reformatted. Adopting it now
# would rewrite the exact byte-strings mutation_check.py matches against the
# files they mutate, which would silently stop those mutations from applying --
# the harness reports them as never having run. That is a real cost against no
# correctness benefit, so the formatter is available and advisory, not required.
format-diff:  # Show what black would change (advisory; not a gate)
	pip install black -q
	black --check --diff tests/ mutation_check.py verify_like_ci.py scan_secrets.py

figures:  # Regenerate the paper's figures from the committed data
	cd paper/honest/repro && for f in make_*.py; do python $$f; done

paper: figures  # Compile the paper PDF (regenerates figures first)
	cd paper/honest && pdflatex -interaction=nonstopmode scoring_bias_v2.tex && \
		bibtex scoring_bias_v2 && \
		pdflatex -interaction=nonstopmode scoring_bias_v2.tex && \
		pdflatex -interaction=nonstopmode scoring_bias_v2.tex

validate:  # Check the paper's prose and figures against the derived data
	cd paper/honest/repro && python check_prose.py && python check_figures.py

docs:  # Build project documentation
	@echo "Building documentation..."
	@if command -v pdoc > /dev/null 2>&1; then \
		pdoc --output-dir docs/api src/scoring_bias/; \
		echo "✓ API docs generated in docs/api/"; \
	else \
		echo "⚠️ pdoc not installed. Install with: pip install pdoc"; \
		echo "   Falling back to markdown docs summary."; \
		@echo "See README.md and paper/ for project documentation."; \
	fi

archive: arxiv-package  # Generate arXiv submission package (alias)

ci: test lint  # Run all CI checks (test + lint)

reproduce-all: setup test paper archive  # Full end-to-end reproduction pipeline

# run-api and run-dashboard are gone. Both served the fabrication-era synthetic
# datasets (results/bias_interaction_synthetic*.csv), which no part of the paper
# of record reads, from scripts that now live under RETRACTED/legacy/scripts/.
# An advertised target that publishes synthetic data does not belong in a repo
# whose paper states that none is used.

check-credentials:  # Scan every commit for credentials
	python scan_secrets.py

health-check: integrity  # Verify project integrity (alias)

clean:  # Remove all build artifacts, caches, and temp files
	find . -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true
	find . -type d -name .ipynb_checkpoints -exec rm -rf {} + 2>/dev/null || true
	find . -type f -name '*.aux' -delete
	find . -type f -name '*.log' -delete
	find . -type f -name '*.out' -delete
	find . -type f -name '*.toc' -delete
	find . -type f -name '*.bbl' -delete
	find . -type f -name '*.blg' -delete
	find . -type f -name '*.pyc' -delete
	rm -rf build/ dist/ *.egg-info/

integrity:  # Fabrication sweep, guard mutations, and history secret scan
	python -m pytest tests/ -q -rs
	python mutation_check.py
	python scan_secrets.py

verify-like-ci:  # Run everything CI runs, locally, under the pinned stack
	python verify_like_ci.py

verify-clean:  # Run the suite in a fresh clone of HEAD
	python verify_clean_clone.py

arxiv-package:  # Build and verify the honest paper's submission archive
	cd paper/honest && python arxiv_package.py