Fix code highlighting on blame page (#36157)

1. Full file highlighting (fix the legacy todo "we should instead
highlight the whole file at once")
    * Fix #24383
2. Correctly covert file content encoding
3. Remove dead code, split large for-loop into small functions/blocks to
make code maintainable

modules/charset/escape.go

@@ -20,14 +20,17 @@ import (
 // RuneNBSP is the codepoint for NBSP
 const RuneNBSP = 0xa0
 
-// EscapeControlHTML escapes the unicode control sequences in a provided html document
+// EscapeControlHTML escapes the Unicode control sequences in a provided html document
 func EscapeControlHTML(html template.HTML, locale translation.Locale, allowed ...rune) (escaped *EscapeStatus, output template.HTML) {
+	if !setting.UI.AmbiguousUnicodeDetection {
+		return &EscapeStatus{}, html
+	}
 	sb := &strings.Builder{}
 	escaped, _ = EscapeControlReader(strings.NewReader(string(html)), sb, locale, allowed...) // err has been handled in EscapeControlReader
 	return escaped, template.HTML(sb.String())
 }
 
-// EscapeControlReader escapes the unicode control sequences in a provided reader of HTML content and writer in a locale and returns the findings as an EscapeStatus
+// EscapeControlReader escapes the Unicode control sequences in a provided reader of HTML content and writer in a locale and returns the findings as an EscapeStatus
 func EscapeControlReader(reader io.Reader, writer io.Writer, locale translation.Locale, allowed ...rune) (escaped *EscapeStatus, err error) {
 	if !setting.UI.AmbiguousUnicodeDetection {
 		_, err = io.Copy(writer, reader)

modules/highlight/highlight.go

@@ -56,7 +56,39 @@ func NewContext() {
 	})
 }
 
-// Code returns a HTML version of code string with chroma syntax highlighting classes and the matched lexer name
+// UnsafeSplitHighlightedLines splits highlighted code into lines preserving HTML tags
+// It always includes '\n', '\n' can appear at the end of each line or in the middle of HTML tags
+// The '\n' is necessary for copying code from web UI to preserve original code lines
+// ATTENTION: It uses the unsafe conversion between string and []byte for performance reason
+// DO NOT make any modification to the returned [][]byte slice items
+func UnsafeSplitHighlightedLines(code template.HTML) (ret [][]byte) {
+	buf := util.UnsafeStringToBytes(string(code))
+	lineCount := bytes.Count(buf, []byte("\n")) + 1
+	ret = make([][]byte, 0, lineCount)
+	nlTagClose := []byte("\n</")
+	for {
+		pos := bytes.IndexByte(buf, '\n')
+		if pos == -1 {
+			if len(buf) > 0 {
+				ret = append(ret, buf)
+			}
+			return ret
+		}
+		// Chroma highlighting output sometimes have "</span>" right after \n, sometimes before.
+		// * "<span>text\n</span>"
+		// * "<span>text</span>\n"
+		if bytes.HasPrefix(buf[pos:], nlTagClose) {
+			pos1 := bytes.IndexByte(buf[pos:], '>')
+			if pos1 != -1 {
+				pos += pos1
+			}
+		}
+		ret = append(ret, buf[:pos+1])
+		buf = buf[pos+1:]
+	}
+}
+
+// Code returns an HTML version of code string with chroma syntax highlighting classes and the matched lexer name
 func Code(fileName, language, code string) (output template.HTML, lexerName string) {
 	NewContext()
 

modules/highlight/highlight_test.go

@@ -181,3 +181,21 @@ c=2`),
 		})
 	}
 }
+
+func TestUnsafeSplitHighlightedLines(t *testing.T) {
+	ret := UnsafeSplitHighlightedLines("")
+	assert.Empty(t, ret)
+
+	ret = UnsafeSplitHighlightedLines("a")
+	assert.Len(t, ret, 1)
+	assert.Equal(t, "a", string(ret[0]))
+
+	ret = UnsafeSplitHighlightedLines("\n")
+	assert.Len(t, ret, 1)
+	assert.Equal(t, "\n", string(ret[0]))
+
+	ret = UnsafeSplitHighlightedLines("<span>a</span>\n<span>b\n</span>")
+	assert.Len(t, ret, 2)
+	assert.Equal(t, "<span>a</span>\n", string(ret[0]))
+	assert.Equal(t, "<span>b\n</span>", string(ret[1]))
+}