intent-eval-lab/pyproject.toml at main · jeremylongshore/intent-eval-lab · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
[project]
name = "intent-eval-lab"
version = "0.3.0"
description = "Intent Eval Lab — methodology repo. The pyproject is only here for pytest/ruff/mypy/coverage config; no Python package is built."
requires-python = ">=3.11"

# ----------------------------------------------------------------------------
# pytest (L3)
# ----------------------------------------------------------------------------

[tool.pytest.ini_options]
minversion = "8.0"
testpaths = [
  "research/phase-a-0-baseline/tests",
  "scripts/tests",
]
addopts = [
  "-v",
  "--strict-markers",
  "--strict-config",
  "--tb=short",
]
markers = [
  "smoke: end-to-end smoke tests (may shell out to subprocess but never hit network)",
]

# ----------------------------------------------------------------------------
# ruff (L2 code lint + format)
# ----------------------------------------------------------------------------

[tool.ruff]
line-length = 120
target-version = "py311"
extend-exclude = [
  ".audit-harness",
  "research/phase-a-0-baseline/results",
]

[tool.ruff.lint]
# Conservative starter rule set; tighten in follow-up PRs as the corpus stabilizes.
select = [
  "E",    # pycodestyle errors
  "W",    # pycodestyle warnings
  "F",    # pyflakes (unused imports, undefined names)
  "I",    # isort (import order)
  "B",    # flake8-bugbear (likely-bug patterns)
  "UP",   # pyupgrade (modern Python idioms)
  "SIM",  # flake8-simplify
]
ignore = [
  "E501",  # line length — handled by ruff format
  "B008",  # function call in default argument (Argparse defaults trip this)
]

[tool.ruff.lint.per-file-ignores]
"research/phase-a-0-baseline/tests/**" = ["B011"]  # pytest assert idioms
"scripts/tests/**" = ["B011"]  # pytest assert idioms

# ----------------------------------------------------------------------------
# mypy (L2 type checking)
# ----------------------------------------------------------------------------

[tool.mypy]
python_version = "3.11"
strict = true
warn_unused_ignores = true
warn_redundant_casts = true
warn_return_any = true
# Phase A.0 runners import third-party SDKs (anthropic, openai) that may not
# ship type stubs in all environments. Allow missing imports for now; tighten
# once stubs are installed in CI.
ignore_missing_imports = true
# Subprocess + dynamic dataclass patterns in the runners trip strict checks
# in a few spots; relax those locally in the affected files via inline
# `# type: ignore` rather than relaxing project-wide.
exclude = [
  ".audit-harness/.*",
  "research/phase-a-0-baseline/results/.*",
]

# ----------------------------------------------------------------------------
# coverage.py (L3 coverage gate + subprocess instrumentation)
# ----------------------------------------------------------------------------

[tool.coverage.run]
source = ["research/phase-a-0-baseline/scripts"]
omit = [
  "research/phase-a-0-baseline/tests/*",
  "*/__pycache__/*",
  ".audit-harness/*",
]
# Subprocess instrumentation: smoke tests invoke runners via subprocess.run.
# parallel=True + concurrency=multiprocessing causes coverage to write a
# separate .coverage.<pid> file per process; `coverage combine` then merges
# them before report. The CI workflow sets COVERAGE_PROCESS_START to the
# pyproject.toml path so child Pythons auto-init.
parallel = true
concurrency = ["multiprocessing"]

[tool.coverage.report]
# LOCAL floor. Locally the suite measures ~66-68% because the cross-repo IS
# validator + fixtures and the model-provider keys are both present, so the
# arm-runner subprocesses actually execute. CI cannot reach either, measures
# ~40%, and therefore passes `--fail-under=0` to switch this off for CI only
# (see .github/workflows/python-tests.yml). Do NOT lower this number to make CI
# green — that would weaken the local gate to paper over an environment limit.
# Ratchet UP in follow-up PRs as un-tested branches land tests.
fail_under = 60
precision = 1
show_missing = true
skip_covered = false
exclude_also = [
  "raise NotImplementedError",
  "if __name__ == .__main__.:",
  "if TYPE_CHECKING:",
  "pragma: no cover",
]

[tool.coverage.paths]
source = ["research/phase-a-0-baseline/scripts"]