inspect-coco/Taskfile.yml at main · kameshsampath/inspect-coco · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
version: "3"

vars:
    PYTHON_VERSION: "3.12"

tasks:
    default:
        desc: Show available tasks
        cmds:
            - task --list

    # --- Quickstart ---

    quickstart:
        desc: Install, run the hello-world eval, and open results
        cmds:
            - task: install
            - uv run inspect-coco run examples/hello-world --epochs=3
            - uv run inspect view

    # --- Development ---

    install:
        desc: Install project with dev and docs dependencies
        cmds:
            - uv sync --group dev --group docs

    lint:
        desc: Run linters (ruff check + format check)
        cmds:
            - uv run ruff check src/ tests/
            - uv run ruff format --check src/ tests/

    fmt:
        desc: Auto-format code
        cmds:
            - uv run ruff format src/ tests/
            - uv run ruff check --fix src/ tests/

    typecheck:
        desc: Run type checker
        cmds:
            - uv run pyright

    test:
        desc: Run tests
        cmds:
            - uv run pytest tests/ {{.CLI_ARGS}}

    test:quick:
        desc: Run tests excluding Docker-dependent ones
        cmds:
            - uv run pytest tests/ -m "not docker" {{.CLI_ARGS}}

    check:
        desc: Run all checks (lint + typecheck + test)
        cmds:
            - task: lint
            - task: typecheck
            - task: test

    # --- Docs ---

    docs:serve:
        desc: Serve docs locally with live reload
        cmds:
            - uv run mkdocs serve

    docs:build:
        desc: Build static docs site
        cmds:
            - uv run mkdocs build --strict

    docs:deploy:
        desc: Deploy docs to GitHub Pages
        cmds:
            - uv run mkdocs gh-deploy --force

    # --- Eval ---

    eval:scaffold:
        desc: Generate eval tasks from CoCo plugin structure
        cmds:
            - uv run inspect-coco scaffold {{.CLI_ARGS}}

    eval:run:
        desc: Run eval suite on examples
        cmds:
            - uv run inspect-coco run examples/ {{.CLI_ARGS}}

    eval:idd:
        desc: Check IDD scores for all examples
        cmds:
            - uv run inspect-coco idd-check examples/

    eval:dry-run:
        desc: Dry-run eval suite (no Docker needed)
        cmds:
            - uv run inspect-coco run examples/ --dry-run

    eval:view:
        desc: Open Inspect log viewer
        cmds:
            - uv run inspect view

    # --- Build & Release ---

    build:
        desc: Build package wheel
        cmds:
            - uv build

    clean:
        desc: Remove build artifacts
        cmds:
            - rm -rf dist/ site/ .pytest_cache/ .ruff_cache/
            - find . -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true