1
0
Fork 0
mirror of https://codeberg.org/forgejo/forgejo.git synced 2025-07-23 11:39:41 +02:00

[v12.0/forgejo] fix(code-search): HighlightSearchResultCode should count the number of bytes and not the number of runes (#8498)

**Backport:** https://codeberg.org/forgejo/forgejo/pulls/8492

fixes incorrect handling of unicode in the matched line

Co-authored-by: Shiny Nematoda <snematoda.751k2@aleeas.com>
Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/8498
Co-authored-by: forgejo-backport-action <forgejo-backport-action@noreply.codeberg.org>
Co-committed-by: forgejo-backport-action <forgejo-backport-action@noreply.codeberg.org>
This commit is contained in:
forgejo-backport-action 2025-07-12 19:00:07 +02:00 committed by Earl Warren
parent 6e9a2e89e8
commit fc726fae9f
2 changed files with 134 additions and 9 deletions

View file

@ -97,7 +97,7 @@ func HighlightSearchResultCode(filename string, lineNums []int, highlightRanges
conv := hcd.ConvertToPlaceholders(string(hl))
convLines := strings.Split(conv, "\n")
// each highlightRange is of the form [line number, start pos, end pos]
// each highlightRange is of the form [line number, start byte offset, end byte offset]
for _, highlightRange := range highlightRanges {
ln, start, end := highlightRange[0], highlightRange[1], highlightRange[2]
line := convLines[ln]
@ -105,15 +105,18 @@ func HighlightSearchResultCode(filename string, lineNums []int, highlightRanges
continue
}
sr := strings.NewReader(line)
sb := strings.Builder{}
count := -1
isOpen := false
for _, r := range line {
for r, size, err := sr.ReadRune(); err == nil; r, size, err = sr.ReadRune() {
if token, ok := hcd.PlaceholderTokenMap[r];
// token was not found
!ok ||
// token was marked as used
token == "" ||
!ok {
count += size
} else if
// token was marked as used
token == "" ||
// the token is not an valid html tag emitted by chroma
!(len(token) > 6 && (token[0:5] == "<span" || token[0:6] == "</span")) {
count++
@ -132,15 +135,15 @@ func HighlightSearchResultCode(filename string, lineNums []int, highlightRanges
continue
}
switch count {
case end:
switch {
case count >= end:
// if tag is not open, no need to close
if !isOpen {
break
}
sb.WriteRune(endTag)
isOpen = false
case start:
case count >= start:
// if tag is open, do not open again
if isOpen {
break
@ -161,7 +164,7 @@ func HighlightSearchResultCode(filename string, lineNums []int, highlightRanges
highlightedLines := strings.Split(hcd.Recover(conv), "\n")
// The lineNums outputted by highlight.Code might not match the original lineNums, because "highlight" removes the last `\n`
lines := make([]ResultLine, min(len(highlightedLines), len(lineNums)))
for i := 0; i < len(lines); i++ {
for i := range len(lines) {
lines[i].Num = lineNums[i]
lines[i].FormattedContent = template.HTML(highlightedLines[i])
}

View file

@ -0,0 +1,122 @@
// Copyright 2025 The Forgejo Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package code
import (
"html/template"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestHighlightSearchResultCode(t *testing.T) {
opts := []struct {
Title string
File string
Lines []int
Range [][3]int
Code string
Result []template.HTML
}{
{
Title: "One Match Text",
File: "test.txt",
Range: [][3]int{{1, 5, 9}},
Code: "First Line\nMark this only\nThe End",
Result: []template.HTML{
"First Line",
"Mark <span class=\"search-highlight\">this</span> only",
"The End",
},
},
{
Title: "Two Match Text",
File: "test.txt",
Range: [][3]int{
{1, 5, 9},
{2, 5, 9},
},
Code: "First Line\nMark this only\nMark this too\nThe End",
Result: []template.HTML{
"First Line",
"Mark <span class=\"search-highlight\">this</span> only",
"Mark <span class=\"search-highlight\">this</span> too",
"The End",
},
},
{
Title: "Unicode Before",
File: "test.txt",
Range: [][3]int{{1, 10, 14}},
Code: "First Line\nMark 👉 this only\nThe End",
Result: []template.HTML{
"First Line",
"Mark 👉 <span class=\"search-highlight\">this</span> only",
"The End",
},
},
{
Title: "Unicode Between",
File: "test.txt",
Range: [][3]int{{1, 5, 14}},
Code: "First Line\nMark this 😊 only\nThe End",
Result: []template.HTML{
"First Line",
"Mark <span class=\"search-highlight\">this 😊</span> only",
"The End",
},
},
{
Title: "Unicode Before And Between",
File: "test.txt",
Range: [][3]int{{1, 10, 19}},
Code: "First Line\nMark 👉 this 😊 only\nThe End",
Result: []template.HTML{
"First Line",
"Mark 👉 <span class=\"search-highlight\">this 😊</span> only",
"The End",
},
},
{
Title: "Golang",
File: "test.go",
Range: [][3]int{{1, 14, 23}},
Code: "func main() {\n\tfmt.Println(\"mark this\")\n}",
Result: []template.HTML{
"<span class=\"kd\">func</span> <span class=\"nf\">main</span><span class=\"p\">(</span><span class=\"p\">)</span> <span class=\"p\">{</span>",
"\t<span class=\"nx\">fmt</span><span class=\"p\">.</span><span class=\"nf\">Println</span><span class=\"p\">(</span><span class=\"s\">&#34;<span class=\"search-highlight\">mark this</span>&#34;</span><span class=\"p\">)</span>",
"<span class=\"p\">}</span>",
},
},
{
Title: "Golang Unicode",
File: "test.go",
Range: [][3]int{{1, 14, 28}},
Code: "func main() {\n\tfmt.Println(\"mark this 😊\")\n}",
Result: []template.HTML{
"<span class=\"kd\">func</span> <span class=\"nf\">main</span><span class=\"p\">(</span><span class=\"p\">)</span> <span class=\"p\">{</span>",
"\t<span class=\"nx\">fmt</span><span class=\"p\">.</span><span class=\"nf\">Println</span><span class=\"p\">(</span><span class=\"s\">&#34;<span class=\"search-highlight\">mark this 😊</span>&#34;</span><span class=\"p\">)</span>",
"<span class=\"p\">}</span>",
},
},
}
for _, o := range opts {
t.Run(o.Title, func(t *testing.T) {
lines := []int{}
for i := range strings.Count(strings.TrimSuffix(o.Code, "\n"), "\n") + 1 {
lines = append(lines, i+1)
}
res := HighlightSearchResultCode(o.File, lines, o.Range, o.Code)
assert.Len(t, res, len(o.Result))
assert.Len(t, res, len(lines))
for i, r := range res {
require.Equal(t, lines[i], r.Num)
require.Equal(t, o.Result[i], r.FormattedContent)
}
})
}
}