|
1 | | -#!/usr/bin/env sh |
| 1 | +#!/usr/bin/env bash |
2 | 2 |
|
3 | 3 | # Synopsis: |
4 | | -# Run the test runner on a solution. |
| 4 | +# Run the test runner on a solution, producing v3-format results.json. |
| 5 | +# |
| 6 | +# Strategy: post-process the text output from |
| 7 | +# `factor -roots=. -run=exercism-tools <slug>`. Factor's tools.test prints a |
| 8 | +# `Unit Test: <args>` header before each test, the test body's stdout, then |
| 9 | +# `--> test failed!` on failure. After all tests run, exercism-tools prints |
| 10 | +# per-failure blocks bracketed by `###FAIL_BEGIN###`/`###FAIL_END###`, |
| 11 | +# each containing `<path>: <line#>` and either an `=== Expected: / === Got:` |
| 12 | +# diff or a thrown error message. |
| 13 | +# |
| 14 | +# Spec: https://exercism.org/docs/building/tooling/test-runners/interface |
5 | 15 |
|
6 | 16 | # Arguments: |
7 | 17 | # $1: exercise slug |
8 | 18 | # $2: path to solution folder |
9 | 19 | # $3: path to output directory |
10 | 20 |
|
11 | | -# Output: |
12 | | -# Writes the test results to a results.json file in the passed-in output directory. |
13 | | -# The test results are formatted according to the specifications at https://github.com/exercism/docs/blob/main/building/tooling/test-runners/interface.md |
| 21 | +set -euo pipefail |
14 | 22 |
|
15 | | -# Example: |
16 | | -# ./bin/run.sh two-fer path/to/solution/folder/ path/to/output/directory/ |
17 | | - |
18 | | -# If any required arguments is missing, print the usage and exit |
19 | | -if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then |
20 | | - echo "usage: ./bin/run.sh exercise-slug path/to/solution/folder/ path/to/output/directory/" |
| 23 | +if [[ -z "${1:-}" || -z "${2:-}" || -z "${3:-}" ]]; then |
| 24 | + echo "usage: $0 <slug> <solution-dir> <output-dir>" >&2 |
21 | 25 | exit 1 |
22 | 26 | fi |
23 | 27 |
|
24 | 28 | slug="$1" |
25 | 29 | solution_dir=$(realpath "${2%/}") |
26 | 30 | output_dir=$(realpath "${3%/}") |
| 31 | +mkdir -p "$output_dir" |
27 | 32 | results_file="${output_dir}/results.json" |
28 | | - |
29 | | -# Create the output directory if it doesn't exist |
30 | | -mkdir -p "${output_dir}" |
| 33 | +canonical_root="/opt/test-runner/tests/${slug}" |
| 34 | +test_file="${solution_dir}/${slug}/${slug}-tests.factor" |
31 | 35 |
|
32 | 36 | echo "${slug}: testing..." |
33 | 37 |
|
34 | | -# Remove STOP-HERE lines to unskip all tests |
35 | | -sed -i '/STOP-HERE/d' "${solution_dir}/${slug}/${slug}-tests.factor" |
36 | | - |
37 | | -# Run the tests for the provided implementation file and redirect stdout and |
38 | | -# stderr to capture it |
39 | | -test_output=$(cd "${solution_dir}" && factor -e="USING: vocabs.loader tools.test tools.test.private namespaces kernel system ; \".\" add-vocab-root \"${slug}\" require \"${slug}\" test test-failures get empty? [ 0 exit ] [ 1 exit ] if" 2>&1) |
40 | | -test_exit=$? |
41 | | -test_output=$(printf '%s\n' "${test_output}" | grep -v "^fatal error for monitor root" | sed '/^(U) \[/,$d' | sed '/^$/d') |
42 | | - |
43 | | -# Write the results.json file based on the exit code of the command that was |
44 | | -# just executed that tested the implementation file |
45 | | -if [ $test_exit -eq 0 ]; then |
46 | | - jq -n '{version: 1, status: "pass"}' > ${results_file} |
47 | | -else |
48 | | - # OPTIONAL: Sanitize the output |
49 | | - # In some cases, the test output might be overly verbose, in which case stripping |
50 | | - # the unneeded information can be very helpful to the student |
51 | | - # sanitized_test_output=$(printf "${test_output}" | sed -n '/Test results:/,$p') |
52 | | - |
53 | | - # OPTIONAL: Manually add colors to the output to help scanning the output for errors |
54 | | - # If the test output does not contain colors to help identify failing (or passing) |
55 | | - # tests, it can be helpful to manually add colors to the output |
56 | | - # colorized_test_output=$(echo "${test_output}" \ |
57 | | - # | GREP_COLOR='01;31' grep --color=always -E -e '^(ERROR:.*|.*failed)$|$' \ |
58 | | - # | GREP_COLOR='01;32' grep --color=always -E -e '^.*passed$|$') |
59 | | - |
60 | | - jq -n --arg output "${test_output}" '{version: 1, status: "fail", message: $output}' > ${results_file} |
| 38 | +if [[ ! -f "$test_file" ]]; then |
| 39 | + jq -n --arg msg "test file not found: ${slug}-tests.factor" \ |
| 40 | + '{version: 3, status: "error", message: $msg}' >"$results_file" |
| 41 | + exit 0 |
61 | 42 | fi |
62 | 43 |
|
| 44 | +# Copy the fixture to a fresh temp dir so the rewrite below does not mutate |
| 45 | +# the source. |
| 46 | +tmp_dir=$(mktemp -d -t "factor-runner-${slug}-XXXXX") |
| 47 | +trap 'rm -rf "$tmp_dir"' EXIT |
| 48 | +cp -r "${solution_dir}/." "$tmp_dir" |
| 49 | +stripped_tests="${tmp_dir}/${slug}/${slug}-tests.factor" |
| 50 | +awk '!/^STOP-HERE$/' "$stripped_tests" > "${stripped_tests}.new" |
| 51 | +mv "${stripped_tests}.new" "$stripped_tests" |
| 52 | + |
| 53 | +# Run Factor; capture combined stdout/stderr. |
| 54 | +set +e |
| 55 | +raw_output=$(cd "$tmp_dir" && factor -roots=. -run=exercism-tools "$slug" 2>&1) |
| 56 | +set -e |
| 57 | +# Normalize the tmp path to the canonical Docker path. |
| 58 | +raw_output=${raw_output//$tmp_dir/$canonical_root} |
| 59 | + |
| 60 | +# Awk parser shared by all stages: JSON-escape a single string field. |
| 61 | +read -r -d '' AWK_JSON <<'AWK' || true |
| 62 | +function json_str(s, r) { |
| 63 | + r = s |
| 64 | + gsub(/\\/, "\\\\", r) |
| 65 | + gsub(/"/, "\\\"", r) |
| 66 | + gsub(/\b/, "\\b", r) |
| 67 | + gsub(/\f/, "\\f", r) |
| 68 | + gsub(/\n/, "\\n", r) |
| 69 | + gsub(/\r/, "\\r", r) |
| 70 | + gsub(/\t/, "\\t", r) |
| 71 | + return "\"" r "\"" |
| 72 | +} |
| 73 | +AWK |
| 74 | + |
| 75 | +# 1. Extract source-test records (one JSON object per line, NDJSON): |
| 76 | +# {"line_no":N,"task_id":N|null,"test_code":"..."} |
| 77 | +# Reads the post-strip file so line numbers match what Factor reports. |
| 78 | +src_tests=$(awk "$AWK_JSON"' |
| 79 | + BEGIN { task = "null" } |
| 80 | + /^[[:space:]]*TASK:[[:space:]]+[0-9]+/ { |
| 81 | + match($0, /TASK:[[:space:]]+[0-9]+/) |
| 82 | + s = substr($0, RSTART, RLENGTH) |
| 83 | + sub(/^TASK:[[:space:]]+/, "", s) |
| 84 | + task = s |
| 85 | + next |
| 86 | + } |
| 87 | + /(unit-test|unit-test~|unit-test-v~|long-unit-test|must-fail-with|must-fail|must-not-fail|must-infer|must-infer-as)[[:space:]]*$/ { |
| 88 | + line = $0 |
| 89 | + sub(/^[[:space:]]+/, "", line) |
| 90 | + sub(/[[:space:]]+$/, "", line) |
| 91 | + printf "{\"line_no\":%d,\"task_id\":%s,\"test_code\":%s}\n", NR, task, json_str(line) |
| 92 | + } |
| 93 | +' "${tmp_dir}/${slug}/${slug}-tests.factor") |
| 94 | + |
| 95 | +# 2. Parse Factor stdout into NDJSON segments and failures: |
| 96 | +# segments: {"type":"segment","idx":N,"failed":bool,"output":"..."} |
| 97 | +# failures: {"type":"failure","line_no":N,"message":"..."} |
| 98 | +parsed=$(printf '%s\n' "$raw_output" | awk "$AWK_JSON"' |
| 99 | + function close_segment( out, i) { |
| 100 | + if (idx == 0) return |
| 101 | + out = "" |
| 102 | + for (i = 1; i <= seg_n; i++) out = out (i > 1 ? "\n" : "") seg[i] |
| 103 | + sub(/^\n+/, "", out); sub(/\n+$/, "", out) |
| 104 | + printf "{\"type\":\"segment\",\"idx\":%d,\"failed\":%s,\"output\":%s}\n", |
| 105 | + idx, (seg_failed ? "true" : "false"), json_str(out) |
| 106 | + } |
| 107 | + function close_failure( body, i) { |
| 108 | + body = "" |
| 109 | + for (i = 1; i <= fail_n; i++) body = body (i > 1 ? "\n" : "") fail[i] |
| 110 | + sub(/^\n+/, "", body); sub(/\n+$/, "", body) |
| 111 | + printf "{\"type\":\"failure\",\"line_no\":%d,\"message\":%s}\n", |
| 112 | + fail_line, json_str(body) |
| 113 | + } |
| 114 | + # Factor renders each test-word name into a title, e.g.: |
| 115 | + # unit-test → "Unit Test:" |
| 116 | + # unit-test~ → "Unit Test~:" |
| 117 | + # unit-test-v~ → "Unit Test V~:" |
| 118 | + # long-unit-test → "Long Unit Test:" |
| 119 | + # must-fail → "Must Fail:" |
| 120 | + # must-fail-with → "Must Fail With:" |
| 121 | + # must-not-fail → "Must Not Fail:" |
| 122 | + # must-infer → "Must Infer:" |
| 123 | + # must-infer-as → "Must Infer As:" |
| 124 | + BEGIN { |
| 125 | + state = "inline"; idx = 0 |
| 126 | + header_re = "^(Unit Test|Unit Test~|Unit Test V~|Long Unit Test|Must Fail|Must Fail With|Must Not Fail|Must Infer|Must Infer As): " |
| 127 | + } |
| 128 | + state == "inline" && $0 ~ header_re { |
| 129 | + close_segment() |
| 130 | + idx++; seg_failed = 0; seg_n = 0; delete seg |
| 131 | + next |
| 132 | + } |
| 133 | + state == "inline" && $0 == "###FAIL_BEGIN###" { |
| 134 | + close_segment(); idx = 0 |
| 135 | + state = "fail_loc"; next |
| 136 | + } |
| 137 | + state == "inline" && $0 == "--> test failed!" { |
| 138 | + seg_failed = 1; next |
| 139 | + } |
| 140 | + state == "inline" { |
| 141 | + if (idx > 0) { seg_n++; seg[seg_n] = $0 } |
| 142 | + next |
| 143 | + } |
| 144 | + state == "fail_loc" { |
| 145 | + if (match($0, /:[[:space:]]*[0-9]+[[:space:]]*$/)) { |
| 146 | + n = substr($0, RSTART, RLENGTH); gsub(/[^0-9]/, "", n) |
| 147 | + fail_line = n + 0 |
| 148 | + } else { fail_line = 0 } |
| 149 | + fail_n = 0; delete fail |
| 150 | + state = "fail_body"; next |
| 151 | + } |
| 152 | + state == "fail_body" && $0 == "###FAIL_END###" { |
| 153 | + close_failure() |
| 154 | + state = "fail_between"; next |
| 155 | + } |
| 156 | + state == "fail_body" { |
| 157 | + fail_n++; fail[fail_n] = $0; next |
| 158 | + } |
| 159 | + state == "fail_between" && $0 == "###FAIL_BEGIN###" { |
| 160 | + state = "fail_loc"; next |
| 161 | + } |
| 162 | + END { close_segment() } |
| 163 | +') |
| 164 | + |
| 165 | +segments=$(printf '%s\n' "$parsed" | awk '/"type":"segment"/' || true) |
| 166 | +failures=$(printf '%s\n' "$parsed" | awk '/"type":"failure"/' || true) |
| 167 | + |
| 168 | +# 3. If no segments emitted, surface a top-level error from the raw output. |
| 169 | +if [[ -z "$segments" ]]; then |
| 170 | + cleaned=$(printf '%s\n' "$raw_output" | awk '/^\([UO]\) /{exit} {print}' \ |
| 171 | + | awk 'NF { print; blank = 0; next } !blank { print; blank = 1 }') |
| 172 | + if [[ -z "$cleaned" ]]; then cleaned="No tests were executed"; fi |
| 173 | + jq -n --arg msg "$cleaned" '{version:3, status:"error", message:$msg}' >"$results_file" |
| 174 | + exit 0 |
| 175 | +fi |
| 176 | + |
| 177 | +# 4. Compose the v3 JSON. |
| 178 | +# --slurpfile would require files; instead pass NDJSON via --argjson after |
| 179 | +# converting each line. We use jq -s on a pipeline of NDJSON inputs. |
| 180 | +jq -n \ |
| 181 | + --argjson srcs "$(printf '%s\n' "$src_tests" | jq -s '.')" \ |
| 182 | + --argjson segs "$(printf '%s\n' "$segments" | jq -s '.')" \ |
| 183 | + --argjson fails "$(printf '%s\n' "$failures" | jq -s '.')" \ |
| 184 | + ' |
| 185 | + ($fails | map({(.line_no|tostring): .message}) | add // {}) as $fail_by_line |
| 186 | + | $segs | sort_by(.idx) |
| 187 | + | to_entries |
| 188 | + | map( |
| 189 | + .value as $seg |
| 190 | + | (.key) as $i |
| 191 | + | ($srcs[$i] // null) as $src |
| 192 | + | ($src.line_no | tostring) as $ln |
| 193 | + | ($fail_by_line[$ln] // null) as $msg |
| 194 | + | ( |
| 195 | + if $seg.failed then |
| 196 | + if ($msg // "" | startswith("=== Expected:")) then "fail" |
| 197 | + elif $msg then "error" |
| 198 | + else "fail" end |
| 199 | + else "pass" end |
| 200 | + ) as $status |
| 201 | + | { |
| 202 | + name: ("Test " + ((.key + 1) | tostring)), |
| 203 | + status: $status, |
| 204 | + test_code: ($src.test_code // ""), |
| 205 | + } |
| 206 | + + (if $src.task_id then {task_id: $src.task_id} else {} end) |
| 207 | + + (if $seg.failed then {message: ($msg // "test failed")} else {} end) |
| 208 | + + (if $seg.output != "" then {output: ($seg.output[0:500])} else {} end) |
| 209 | + ) |
| 210 | + | (if all(.status == "pass") then "pass" else "fail" end) as $top |
| 211 | + | {version: 3, status: $top, tests: .} |
| 212 | + ' >"$results_file" |
| 213 | + |
63 | 214 | echo "${slug}: done" |
0 commit comments