Skip to content

Commit 05f8f33

Browse files
keiravillekodekeiravillekode
authored
Report the failing line in per-test messages (#23)
Sync exercism-tools from track repo to runner test cases In results.json prefix the message with "<path>:<line#>" Blank STOP-HERE lines rather than deleting them. Add passing and failing tests that use the assocs.extras vocabulary.
1 parent 722b038 commit 05f8f33

29 files changed

Lines changed: 165 additions & 151 deletions

File tree

‎Dockerfile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,8 +41,8 @@ RUN rm -rf .git build vm src misc Factor.app \
4141
GNUmakefile Nmakefile LICENSE.txt README.md \
4242
build.sh build.cmd unmaintained
4343

44-
# Keep only the extra/ source that sequences.extras needs to compile on demand:
45-
# itself plus extra/assocs.extras and extra/shuffle.
44+
# Keep only the extra/ source needed to compile sequences.extras and
45+
# assocs.extras on demand: those two plus extra/shuffle.
4646
RUN find extra -mindepth 1 -maxdepth 1 \
4747
! -name sequences ! -name assocs ! -name shuffle \
4848
-exec rm -rf {} + && \

‎bin/run.sh‎

Lines changed: 22 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,11 @@ tmp_dir=$(mktemp -d -t "factor-runner-${slug}-XXXXX")
4747
trap 'rm -rf "${tmp_dir}"' EXIT
4848
cp -r "${solution_dir}/." "${tmp_dir}"
4949
stripped_tests="${tmp_dir}/${slug}/${slug}-tests.factor"
50-
awk '!/^STOP-HERE$/' "${stripped_tests}" > "${stripped_tests}.new"
50+
# Blank each STOP-HERE line rather than deleting it: a blank line is a no-op
51+
# to Factor, and keeping the line count intact means the line numbers Factor
52+
# reports for failures still match the file the student submitted.
53+
awk '{ if ($0 ~ /^STOP-HERE$/) print ""; else print }' \
54+
"${stripped_tests}" > "${stripped_tests}.new"
5155
mv "${stripped_tests}.new" "${stripped_tests}"
5256

5357
# Run Factor; capture combined stdout/stderr.
@@ -131,7 +135,7 @@ src_tests=$(awk "${awk_json}"'
131135

132136
# 2. Parse Factor stdout into NDJSON segments and failures:
133137
# segments: {"type":"segment","idx":N,"failed":bool,"name":"...","output":"..."}
134-
# failures: {"type":"failure","line_no":N,"message":"..."}
138+
# failures: {"type":"failure","line_no":N,"location":"...","message":"..."}
135139
parsed=$(printf '%s\n' "${raw_output}" | awk "${awk_json}"'
136140
function close_segment( out, i) {
137141
if (idx == 0) return
@@ -147,8 +151,8 @@ parsed=$(printf '%s\n' "${raw_output}" | awk "${awk_json}"'
147151
for (i = 1; i <= fail_n; i++) body = body (i > 1 ? "\n" : "") fail[i]
148152
sub(/^\n+/, "", body)
149153
sub(/\n+$/, "", body)
150-
printf "{\"type\":\"failure\",\"line_no\":%d,\"message\":%s}\n",
151-
fail_line, json_str(body)
154+
printf "{\"type\":\"failure\",\"line_no\":%d,\"location\":%s,\"message\":%s}\n",
155+
fail_line, json_str(fail_loc), json_str(body)
152156
}
153157
# Factor renders each test-word name into a title, e.g.:
154158
# unit-test → "Unit Test:"
@@ -221,8 +225,14 @@ parsed=$(printf '%s\n' "${raw_output}" | awk "${awk_json}"'
221225
n = substr($0, RSTART, RLENGTH)
222226
gsub(/[^0-9]/, "", n)
223227
fail_line = n + 0
228+
# exercism-tools prints "<path>: <line#>"; collapse it to the
229+
# conventional "<path>:<line#>" so the student can paste it
230+
# into an editor. The path is already relative to the solution
231+
# root, so it is safe to show as-is.
232+
fail_loc = substr($0, 1, RSTART - 1) ":" fail_line
224233
} else {
225234
fail_line = 0
235+
fail_loc = ""
226236
}
227237
fail_n = 0
228238
delete fail
@@ -276,15 +286,16 @@ jq -n \
276286
--argjson segs "$(printf '%s\n' "${segments}" | jq -s '.')" \
277287
--argjson fails "$(printf '%s\n' "${failures}" | jq -s '.')" \
278288
'
279-
($fails | map({(.line_no|tostring): .message}) | add // {}) as $fail_by_line
289+
($fails | map({(.line_no|tostring): .}) | add // {}) as $fail_by_line
280290
| $segs | sort_by(.idx)
281291
| to_entries
282292
| map(
283293
.value as $seg
284294
| (.key) as $i
285295
| ($srcs[$i] // null) as $src
286296
| ($src.line_no | tostring) as $ln
287-
| ($fail_by_line[$ln] // null) as $msg
297+
| ($fail_by_line[$ln] // null) as $fail
298+
| ($fail.message // null) as $msg
288299
| (
289300
if $seg.failed then
290301
if ($msg // "" | startswith("=== Expected:")) then "fail"
@@ -300,7 +311,11 @@ jq -n \
300311
test_code: ($src.test_code // ""),
301312
}
302313
+ (if $src.task_id then {task_id: $src.task_id} else {} end)
303-
+ (if $seg.failed then {message: ($msg // "test failed")} else {} end)
314+
+ (if $seg.failed then
315+
{message: (if $msg == null then "test failed"
316+
elif ($fail.location // "") == "" then $msg
317+
else $fail.location + "\n" + $msg end)}
318+
else {} end)
304319
+ (if $seg.output != "" then {output: ($seg.output[0:500])} else {} end)
305320
)
306321
| (if all(.status == "pass") then "pass" else "fail" end) as $top

‎tests/all-fail/exercism-tools/exercism-tools.factor‎

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
USING: accessors command-line continuations debugger io kernel
2-
lexer namespaces sequences source-files.errors.debugger
3-
system tools.test vocabs vocabs.loader ;
2+
lexer namespaces prettyprint.config sequences
3+
source-files.errors.debugger system tools.test vocabs
4+
vocabs.loader ;
45
IN: exercism-tools
56

67
SYNTAX: STOP-HERE
@@ -9,26 +10,23 @@ SYNTAX: STOP-HERE
910
SYNTAX: TASK:
1011
lexer get next-line ;
1112

12-
! Label the test that follows with its description. The marker lets the
13-
! wrapper strip this line from captured output and attach it to the next
14-
! test as a name, rather than leaving it in the previous test's output.
13+
! Label the test that follows with its description.
1514
: description ( str -- )
1615
"###DESC### " write print ;
1716

18-
! Print one failure block in a stable, parser-friendly form. Bracketed by
19-
! markers so a wrapper can split the stream reliably and avoid Factor's
20-
! noisy callstack output (which is interleaved with subsequent failures).
17+
! Print one failure block in a stable, parser-friendly form.
2118
:: print-failure ( failure -- )
2219
"###FAIL_BEGIN###" print
2320
failure error-location print
24-
failure error>> [ error. ] [ 2drop ] recover
21+
[ failure error>> [ error. ] [ 2drop ] recover ] without-limits
2522
"###FAIL_END###" print
2623
flush ;
2724

2825
: print-failures ( -- )
2926
test-failures get [ print-failure ] each ;
3027

3128
: run-exercism-tests ( -- )
29+
vocab-roots [ "." prefix ] change-global
3230
command-line get first
3331
[ require ] [ test ] bi
3432
test-failures get empty?

‎tests/all-fail/expected_results.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,13 +6,13 @@
66
"name": "greet returns hello",
77
"status": "fail",
88
"test_code": "{ \"hello\" } [ greet ] unit-test",
9-
"message": "=== Expected:\n\"hello\"\n=== Got:\n\"wrong\""
9+
"message": "all-fail/all-fail-tests.factor:5\n=== Expected:\n\"hello\"\n=== Got:\n\"wrong\""
1010
},
1111
{
1212
"name": "greet returns world",
1313
"status": "fail",
1414
"test_code": "{ \"world\" } [ greet ] unit-test",
15-
"message": "=== Expected:\n\"world\"\n=== Got:\n\"wrong\""
15+
"message": "all-fail/all-fail-tests.factor:10\n=== Expected:\n\"world\"\n=== Got:\n\"wrong\""
1616
}
1717
]
1818
}

‎tests/concept-concurrency/exercism-tools/exercism-tools.factor‎

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
USING: accessors command-line continuations debugger io kernel
2-
lexer namespaces sequences source-files.errors.debugger
3-
system tools.test vocabs vocabs.loader ;
2+
lexer namespaces prettyprint.config sequences
3+
source-files.errors.debugger system tools.test vocabs
4+
vocabs.loader ;
45
IN: exercism-tools
56

67
SYNTAX: STOP-HERE
@@ -9,26 +10,23 @@ SYNTAX: STOP-HERE
910
SYNTAX: TASK:
1011
lexer get next-line ;
1112

12-
! Label the test that follows with its description. The marker lets the
13-
! wrapper strip this line from captured output and attach it to the next
14-
! test as a name, rather than leaving it in the previous test's output.
13+
! Label the test that follows with its description.
1514
: description ( str -- )
1615
"###DESC### " write print ;
1716

18-
! Print one failure block in a stable, parser-friendly form. Bracketed by
19-
! markers so a wrapper can split the stream reliably and avoid Factor's
20-
! noisy callstack output (which is interleaved with subsequent failures).
17+
! Print one failure block in a stable, parser-friendly form.
2118
:: print-failure ( failure -- )
2219
"###FAIL_BEGIN###" print
2320
failure error-location print
24-
failure error>> [ error. ] [ 2drop ] recover
21+
[ failure error>> [ error. ] [ 2drop ] recover ] without-limits
2522
"###FAIL_END###" print
2623
flush ;
2724

2825
: print-failures ( -- )
2926
test-failures get [ print-failure ] each ;
3027

3128
: run-exercism-tests ( -- )
29+
vocab-roots [ "." prefix ] change-global
3230
command-line get first
3331
[ require ] [ test ] bi
3432
test-failures get empty?

‎tests/concept-multiline/exercism-tools/exercism-tools.factor‎

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
USING: accessors command-line continuations debugger io kernel
2-
lexer namespaces sequences source-files.errors.debugger
3-
system tools.test vocabs vocabs.loader ;
2+
lexer namespaces prettyprint.config sequences
3+
source-files.errors.debugger system tools.test vocabs
4+
vocabs.loader ;
45
IN: exercism-tools
56

67
SYNTAX: STOP-HERE
@@ -9,26 +10,23 @@ SYNTAX: STOP-HERE
910
SYNTAX: TASK:
1011
lexer get next-line ;
1112

12-
! Label the test that follows with its description. The marker lets the
13-
! wrapper strip this line from captured output and attach it to the next
14-
! test as a name, rather than leaving it in the previous test's output.
13+
! Label the test that follows with its description.
1514
: description ( str -- )
1615
"###DESC### " write print ;
1716

18-
! Print one failure block in a stable, parser-friendly form. Bracketed by
19-
! markers so a wrapper can split the stream reliably and avoid Factor's
20-
! noisy callstack output (which is interleaved with subsequent failures).
17+
! Print one failure block in a stable, parser-friendly form.
2118
:: print-failure ( failure -- )
2219
"###FAIL_BEGIN###" print
2320
failure error-location print
24-
failure error>> [ error. ] [ 2drop ] recover
21+
[ failure error>> [ error. ] [ 2drop ] recover ] without-limits
2522
"###FAIL_END###" print
2623
flush ;
2724

2825
: print-failures ( -- )
2926
test-failures get [ print-failure ] each ;
3027

3128
: run-exercism-tests ( -- )
29+
vocab-roots [ "." prefix ] change-global
3230
command-line get first
3331
[ require ] [ test ] bi
3432
test-failures get empty?

‎tests/concept-multiline/expected_results.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@
7373
"status": "fail",
7474
"test_code": "{ 6 } [ 4 [ 6 roll-die ] play-seeded ] unit-test",
7575
"task_id": 5,
76-
"message": "=== Expected:\n6\n=== Got:\n2"
76+
"message": "concept-multiline/concept-multiline-tests.factor:62\n=== Expected:\n6\n=== Got:\n2"
7777
},
7878
{
7979
"name": "Test 13",

‎tests/concept-not-parsing/exercism-tools/exercism-tools.factor‎

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
USING: accessors command-line continuations debugger io kernel
2-
lexer namespaces sequences source-files.errors.debugger
3-
system tools.test vocabs vocabs.loader ;
2+
lexer namespaces prettyprint.config sequences
3+
source-files.errors.debugger system tools.test vocabs
4+
vocabs.loader ;
45
IN: exercism-tools
56

67
SYNTAX: STOP-HERE
@@ -9,26 +10,23 @@ SYNTAX: STOP-HERE
910
SYNTAX: TASK:
1011
lexer get next-line ;
1112

12-
! Label the test that follows with its description. The marker lets the
13-
! wrapper strip this line from captured output and attach it to the next
14-
! test as a name, rather than leaving it in the previous test's output.
13+
! Label the test that follows with its description.
1514
: description ( str -- )
1615
"###DESC### " write print ;
1716

18-
! Print one failure block in a stable, parser-friendly form. Bracketed by
19-
! markers so a wrapper can split the stream reliably and avoid Factor's
20-
! noisy callstack output (which is interleaved with subsequent failures).
17+
! Print one failure block in a stable, parser-friendly form.
2118
:: print-failure ( failure -- )
2219
"###FAIL_BEGIN###" print
2320
failure error-location print
24-
failure error>> [ error. ] [ 2drop ] recover
21+
[ failure error>> [ error. ] [ 2drop ] recover ] without-limits
2522
"###FAIL_END###" print
2623
flush ;
2724

2825
: print-failures ( -- )
2926
test-failures get [ print-failure ] each ;
3027

3128
: run-exercism-tests ( -- )
29+
vocab-roots [ "." prefix ] change-global
3230
command-line get first
3331
[ require ] [ test ] bi
3432
test-failures get empty?

‎tests/concept-not-parsing/expected_results.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
{
22
"version": 3,
33
"status": "error",
4-
"message": "/opt/test-runner/tests/concept-not-parsing/concept-not-parsing/concept-not-parsing.factor\n\n6: : cube ( n -- n^3 ) ?!?! ;\n ^\nNo word named '?!?!' found in current vocabulary search path"
4+
"message": "concept-not-parsing/concept-not-parsing.factor\n\n6: : cube ( n -- n^3 ) ?!?! ;\n ^\nNo word named '?!?!' found in current vocabulary search path"
55
}

‎tests/concept-partial-fail/exercism-tools/exercism-tools.factor‎

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
USING: accessors command-line continuations debugger io kernel
2-
lexer namespaces sequences source-files.errors.debugger
3-
system tools.test vocabs vocabs.loader ;
2+
lexer namespaces prettyprint.config sequences
3+
source-files.errors.debugger system tools.test vocabs
4+
vocabs.loader ;
45
IN: exercism-tools
56

67
SYNTAX: STOP-HERE
@@ -9,26 +10,23 @@ SYNTAX: STOP-HERE
910
SYNTAX: TASK:
1011
lexer get next-line ;
1112

12-
! Label the test that follows with its description. The marker lets the
13-
! wrapper strip this line from captured output and attach it to the next
14-
! test as a name, rather than leaving it in the previous test's output.
13+
! Label the test that follows with its description.
1514
: description ( str -- )
1615
"###DESC### " write print ;
1716

18-
! Print one failure block in a stable, parser-friendly form. Bracketed by
19-
! markers so a wrapper can split the stream reliably and avoid Factor's
20-
! noisy callstack output (which is interleaved with subsequent failures).
17+
! Print one failure block in a stable, parser-friendly form.
2118
:: print-failure ( failure -- )
2219
"###FAIL_BEGIN###" print
2320
failure error-location print
24-
failure error>> [ error. ] [ 2drop ] recover
21+
[ failure error>> [ error. ] [ 2drop ] recover ] without-limits
2522
"###FAIL_END###" print
2623
flush ;
2724

2825
: print-failures ( -- )
2926
test-failures get [ print-failure ] each ;
3027

3128
: run-exercism-tests ( -- )
29+
vocab-roots [ "." prefix ] change-global
3230
command-line get first
3331
[ require ] [ test ] bi
3432
test-failures get empty?

0 commit comments

Comments
 (0)