eval/benchmark.json at main · frontier-harness-eval/eval · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
{
  "name": "FrontierHarness Eval v1",
  "status": "frozen",
  "model": "Kimi K3",
  "model_provider": "Fireworks",
  "task_count": 30,
  "harness_count": 9,
  "configuration_count": 12,
  "evaluation_count": 360,
  "canonical_selection": "first valid attempt 1",
  "excluded_labels": [
    "warm",
    "smoke"
  ],
  "task_sources": {
    "terminal_bench": {
      "dataset": "terminal-bench/terminal-bench-2-1",
      "count": 21
    },
    "deep_swe": {
      "repository": "datacurve-ai/deep-swe",
      "version": "v1.1",
      "reproduction_ref": "435ee89ec2f2e2289f33b0da4f992f0b7b7266b9",
      "reproduction_note": "v1.1 is the published dataset label, not an upstream Git tag. The reproduction ref uses separate-verifier images that differ from the frozen public task metadata; validate comparability with a control run.",
      "count": 9
    }
  },
  "harness_configurations": [
    "claude-code",
    "codex",
    "dsh-creator",
    "dsh-minimal",
    "dsh-ptc",
    "dsh-standard",
    "exo",
    "hermes",
    "kimi-code",
    "oh-my-pi",
    "opencode",
    "pi-responses"
  ],
  "results": "results/eval-data.json",
  "interactive_report": "https://frontierharness.org",
  "task_ids": [
    "datacurve/anko-typed-variable-bindings",
    "datacurve/arktype-json-schema-refs-dependencies",
    "datacurve/expr-try-catch-errors",
    "datacurve/fastapi-deprecation-response-headers",
    "datacurve/httpx-multipart-response-parsing",
    "datacurve/katex-multicolumn-array-spans",
    "datacurve/meriyah-explicit-resource-declarations",
    "datacurve/python-statemachine-state-data-scoping",
    "datacurve/scc-bounded-memory-spilling",
    "terminal-bench/build-cython-ext",
    "terminal-bench/chess-best-move",
    "terminal-bench/code-from-image",
    "terminal-bench/constraints-scheduling",
    "terminal-bench/db-wal-recovery",
    "terminal-bench/dna-insert",
    "terminal-bench/extract-elf",
    "terminal-bench/gcode-to-text",
    "terminal-bench/git-leak-recovery",
    "terminal-bench/kv-store-grpc",
    "terminal-bench/largest-eigenval",
    "terminal-bench/log-summary-date-ranges",
    "terminal-bench/merge-diff-arc-agi-task",
    "terminal-bench/modernize-scientific-stack",
    "terminal-bench/multi-source-data-merger",
    "terminal-bench/openssl-selfsigned-cert",
    "terminal-bench/polyglot-c-py",
    "terminal-bench/regex-log",
    "terminal-bench/sanitize-git-repo",
    "terminal-bench/sqlite-db-truncate",
    "terminal-bench/vulnerable-secret"
  ]
}