Merge pull request #1157 from VXNCXNX:fix/case-insensitive-search-offsets

fix: case-insensitive search highlights the wrong bytes

Author: noborus

oviewer/search.go

@@ -13,6 +13,7 @@ import (
 	"strings"
 	"sync/atomic"
 	"unicode"
+	"unicode/utf8"
 
 	"codeberg.org/tslocum/cbind"
 	"github.com/gdamore/tcell/v3"
@@ -50,9 +51,51 @@ func (substr searchWord) MatchString(target string) bool {
 }
 
 // searchWord FindAll searches for strings and returns the index of the match.
+// The returned indexes are positions in target, not in its lowercased form,
+// because lowercasing can change the byte length of a rune (İ, K, ẞ...).
 func (substr searchWord) FindAll(target string) [][]int {
-	target = strings.ToLower(target)
-	return allStringIndex(target, substr.word)
+	lower, offsets := toLowerWithOffsets(target)
+	indexes := allStringIndex(lower, substr.word)
+	if offsets == nil {
+		return indexes
+	}
+	for _, idx := range indexes {
+		idx[0] = offsets[idx[0]]
+		idx[1] = offsets[idx[1]]
+	}
+	return indexes
+}
+
+// toLowerWithOffsets lowercases s the same way [strings.ToLower] does and also
+// returns a table that maps every byte offset of the result back to a byte
+// offset of s. The table is nil when s is ASCII only, since the offsets are
+// then identical and no mapping is needed.
+func toLowerWithOffsets(s string) (string, []int) {
+	if isASCII(s) {
+		return strings.ToLower(s), nil
+	}
+
+	lower := make([]byte, 0, len(s))
+	offsets := make([]int, 0, len(s)+1)
+	for i, r := range s {
+		n := len(lower)
+		lower = utf8.AppendRune(lower, unicode.ToLower(r))
+		for range len(lower) - n {
+			offsets = append(offsets, i)
+		}
+	}
+	offsets = append(offsets, len(s))
+	return string(lower), offsets
+}
+
+// isASCII returns true if s consists of ASCII bytes only.
+func isASCII(s string) bool {
+	for i := range len(s) {
+		if s[i] >= utf8.RuneSelf {
+			return false
+		}
+	}
+	return true
 }
 
 // searchWord String returns the search word.

oviewer/search_test.go

@@ -848,13 +848,55 @@ func Test_sensitive_FindAll(t *testing.T) {
 			},
 			want: [][]int{{0, 1}, {2, 3}, {4, 5}},
 		},
+		{
+			// "İ" is 2 bytes but lowercases to the 1 byte "i",
+			// so the match position must still refer to the original string.
+			name: "testShrinkOnLower",
+			fields: fields{
+				searchWord:    "error",
+				searchReg:     regexpCompile("error", false),
+				caseSensitive: false,
+				regexpSearch:  false,
+			},
+			args: args{
+				s: "İİ error here",
+			},
+			want: [][]int{{5, 10}},
+		},
+		{
+			// "Ⱥ" is 2 bytes but lowercases to the 3 byte "ⱥ".
+			name: "testGrowOnLower",
+			fields: fields{
+				searchWord:    "error",
+				searchReg:     regexpCompile("error", false),
+				caseSensitive: false,
+				regexpSearch:  false,
+			},
+			args: args{
+				s: "Ⱥ error here",
+			},
+			want: [][]int{{3, 8}},
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			substr := NewSearcher(tt.fields.searchWord, tt.fields.searchReg, tt.fields.caseSensitive, tt.fields.regexpSearch)
-			if got := substr.FindAll(tt.args.s); !reflect.DeepEqual(got, tt.want) {
+			got := substr.FindAll(tt.args.s)
+			if !reflect.DeepEqual(got, tt.want) {
 				t.Errorf("sensitiveWord.FindAll() = %v, want %v", got, tt.want)
 			}
+			for _, idx := range got {
+				if tt.fields.regexpSearch {
+					break
+				}
+				if idx[1] > len(tt.args.s) {
+					t.Errorf("sensitiveWord.FindAll() index %v out of range of %q", idx, tt.args.s)
+					continue
+				}
+				if matched := tt.args.s[idx[0]:idx[1]]; !strings.EqualFold(matched, tt.fields.searchWord) {
+					t.Errorf("sensitiveWord.FindAll() matched %q, want %q", matched, tt.fields.searchWord)
+				}
+			}
 		})
 	}
 }