Preserve multibyte characters when wrapping long source lines (#8520)

* Preserve multibyte characters when wrapping long source lines in compiler code frames

* Changelog

Author: fhammerschmidt

CHANGELOG.md

@@ -22,6 +22,8 @@
 
 #### :bug: Bug fix
 
+- Preserve multibyte characters when wrapping long source lines in compiler code frames. https://github.com/rescript-lang/rescript/pull/8520
+
 #### :memo: Documentation
 
 #### :nail_care: Polish

compiler/ml/code_frame.ml

@@ -37,14 +37,23 @@ let leading_space_count str =
   loop 0 0
 
 let break_long_line max_width line =
+  let line_length = String.length line in
+  let rec find_chunk_end pos remaining_width =
+    if pos = line_length || remaining_width = 0 then pos
+    else
+      let char_length =
+        String.get_utf_8_uchar line pos |> Uchar.utf_decode_length
+      in
+      find_chunk_end (pos + char_length) (remaining_width - 1)
+  in
   let rec loop pos accum =
-    if pos = String.length line then accum
+    if pos = line_length then List.rev accum
     else
-      let chunk_length = min max_width (String.length line - pos) in
-      let chunk = String.sub line pos chunk_length in
-      loop (pos + chunk_length) (chunk :: accum)
+      let chunk_end = find_chunk_end pos max_width in
+      let chunk = String.sub line pos (chunk_end - pos) in
+      loop chunk_end (chunk :: accum)
   in
-  loop 0 [] |> List.rev
+  loop 0 []
 
 let filter_mapi f l =
   let rec loop f l i accum =

rewatch/tests/compile/19-utf8-warning.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+
+cd $(dirname $0)
+source "../utils.sh"
+
+bold "Test: Compiler warnings preserve UTF-8 when wrapping source lines"
+
+fixture=$(mktemp -d 2>/dev/null || mktemp -d -t rewatch-utf8-warning)
+trap "rm -rf '$fixture'" EXIT
+
+mkdir -p "$fixture/src"
+
+cat > "$fixture/package.json" <<'EOF'
+{
+  "name": "rewatch-utf8-warning",
+  "version": "0.0.1"
+}
+EOF
+
+cat > "$fixture/rescript.json" <<'EOF'
+{
+  "name": "rewatch-utf8-warning",
+  "sources": { "dir": "src" },
+  "warnings": { "number": "+27" }
+}
+EOF
+
+cat > "$fixture/src/Main.res" <<'EOF'
+let f = (~a, ~b, ~c) => ()
+
+@doc("generic placeholder text with no real source disclosed here ....—trailing text so the line is long enough to wrap onto a second row")
+let _g = 1
+EOF
+
+cd "$fixture"
+compiler_output=$(rewatch build 2>&1)
+build_status=$?
+
+if [ $build_status -ne 0 ]; then
+  error "Build failed"
+  printf "%s\n" "$compiler_output" >&2
+  exit 1
+fi
+
+if ! echo "$compiler_output" | grep -qF '—'; then
+  error "Wrapped warning corrupted the em dash"
+  printf "%s\n" "$compiler_output" >&2
+  exit 1
+fi
+
+if echo "$compiler_output" | grep -qF '�'; then
+  error "Wrapped warning contains a Unicode replacement character"
+  printf "%s\n" "$compiler_output" >&2
+  exit 1
+fi
+
+success "Wrapped warning preserves the em dash"

rewatch/tests/suite.sh

@@ -81,6 +81,7 @@ fi
 ./compile/13-no-infinite-loop-with-cycle.sh &&
 ./compile/17-prod-flag.sh &&
 ./compile/18-external-dep-uncurried-dot.sh &&
+./compile/19-utf8-warning.sh &&
 ./compile/14-no-testrepo-changes.sh &&
 ./compile/15-no-new-files.sh &&
 ./compile/16-snapshots-unchanged.sh &&

tests/ounit_tests/ounit_utf8_test.ml

@@ -4,6 +4,7 @@
 let ( >:: ), ( >::: ) = OUnit.(( >:: ), ( >::: ))
 
 let ( =~ ) = OUnit.assert_equal
+
 let suites =
   __FILE__
   >::: [
@@ -29,4 +30,6 @@ let suites =
                 105;
               ] );
          (__LOC__ >:: fun _ -> Ext_utf8.decode_utf8_string "" =~ []);
+         ( __LOC__ >:: fun _ ->
+           Code_frame.break_long_line 4 "abc—def" =~ ["abc—"; "def"] );
        ]