Add warning for BIDI characters in page renders and in diffs (#17562)
Fix #17514 Given the comments I've adjusted this somewhat. The numbers of characters detected are increased and include things like the use of U+300 to make à instead of à and non-breaking spaces. There is a button which can be used to escape the content to show it. Signed-off-by: Andrew Thornton <art27@cantab.net> Co-authored-by: Gwyneth Morgan <gwymor@tilde.club> Co-authored-by: silverwind <me@silverwind.io> Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>tokarchuk/v1.17
zeripath
3 years ago
committed by
GitHub
parent
ee60f27aec
commit
21ed4fd8da
@ -0,0 +1,230 @@ |
|||||||
|
// Copyright 2021 The Gitea Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a MIT-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
package charset |
||||||
|
|
||||||
|
import ( |
||||||
|
"bytes" |
||||||
|
"fmt" |
||||||
|
"io" |
||||||
|
"strings" |
||||||
|
"unicode" |
||||||
|
"unicode/utf8" |
||||||
|
|
||||||
|
"golang.org/x/text/unicode/bidi" |
||||||
|
) |
||||||
|
|
||||||
|
// EscapeStatus represents the findings of the unicode escaper
|
||||||
|
type EscapeStatus struct { |
||||||
|
Escaped bool |
||||||
|
HasError bool |
||||||
|
HasBadRunes bool |
||||||
|
HasControls bool |
||||||
|
HasSpaces bool |
||||||
|
HasMarks bool |
||||||
|
HasBIDI bool |
||||||
|
BadBIDI bool |
||||||
|
HasRTLScript bool |
||||||
|
HasLTRScript bool |
||||||
|
} |
||||||
|
|
||||||
|
// Or combines two EscapeStatus structs into one representing the conjunction of the two
|
||||||
|
func (status EscapeStatus) Or(other EscapeStatus) EscapeStatus { |
||||||
|
st := status |
||||||
|
st.Escaped = st.Escaped || other.Escaped |
||||||
|
st.HasError = st.HasError || other.HasError |
||||||
|
st.HasBadRunes = st.HasBadRunes || other.HasBadRunes |
||||||
|
st.HasControls = st.HasControls || other.HasControls |
||||||
|
st.HasSpaces = st.HasSpaces || other.HasSpaces |
||||||
|
st.HasMarks = st.HasMarks || other.HasMarks |
||||||
|
st.HasBIDI = st.HasBIDI || other.HasBIDI |
||||||
|
st.BadBIDI = st.BadBIDI || other.BadBIDI |
||||||
|
st.HasRTLScript = st.HasRTLScript || other.HasRTLScript |
||||||
|
st.HasLTRScript = st.HasLTRScript || other.HasLTRScript |
||||||
|
return st |
||||||
|
} |
||||||
|
|
||||||
|
// EscapeControlString escapes the unicode control sequences in a provided string and returns the findings as an EscapeStatus and the escaped string
|
||||||
|
func EscapeControlString(text string) (EscapeStatus, string) { |
||||||
|
sb := &strings.Builder{} |
||||||
|
escaped, _ := EscapeControlReader(strings.NewReader(text), sb) |
||||||
|
return escaped, sb.String() |
||||||
|
} |
||||||
|
|
||||||
|
// EscapeControlBytes escapes the unicode control sequences a provided []byte and returns the findings as an EscapeStatus and the escaped []byte
|
||||||
|
func EscapeControlBytes(text []byte) (EscapeStatus, []byte) { |
||||||
|
buf := &bytes.Buffer{} |
||||||
|
escaped, _ := EscapeControlReader(bytes.NewReader(text), buf) |
||||||
|
return escaped, buf.Bytes() |
||||||
|
} |
||||||
|
|
||||||
|
// EscapeControlReader escapes the unicode control sequences a provided Reader writing the escaped output to the output and returns the findings as an EscapeStatus and an error
|
||||||
|
func EscapeControlReader(text io.Reader, output io.Writer) (escaped EscapeStatus, err error) { |
||||||
|
buf := make([]byte, 4096) |
||||||
|
readStart := 0 |
||||||
|
var n int |
||||||
|
var writePos int |
||||||
|
|
||||||
|
lineHasBIDI := false |
||||||
|
lineHasRTLScript := false |
||||||
|
lineHasLTRScript := false |
||||||
|
|
||||||
|
readingloop: |
||||||
|
for err == nil { |
||||||
|
n, err = text.Read(buf[readStart:]) |
||||||
|
bs := buf[:n+readStart] |
||||||
|
i := 0 |
||||||
|
|
||||||
|
for i < len(bs) { |
||||||
|
r, size := utf8.DecodeRune(bs[i:]) |
||||||
|
// Now handle the codepoints
|
||||||
|
switch { |
||||||
|
case r == utf8.RuneError: |
||||||
|
if writePos < i { |
||||||
|
if _, err = output.Write(bs[writePos:i]); err != nil { |
||||||
|
escaped.HasError = true |
||||||
|
return |
||||||
|
} |
||||||
|
writePos = i |
||||||
|
} |
||||||
|
// runes can be at most 4 bytes - so...
|
||||||
|
if len(bs)-i <= 3 { |
||||||
|
// if not request more data
|
||||||
|
copy(buf, bs[i:]) |
||||||
|
readStart = n - i |
||||||
|
writePos = 0 |
||||||
|
continue readingloop |
||||||
|
} |
||||||
|
// this is a real broken rune
|
||||||
|
escaped.HasBadRunes = true |
||||||
|
escaped.Escaped = true |
||||||
|
if err = writeBroken(output, bs[i:i+size]); err != nil { |
||||||
|
escaped.HasError = true |
||||||
|
return |
||||||
|
} |
||||||
|
writePos += size |
||||||
|
case r == '\n': |
||||||
|
if lineHasBIDI && !lineHasRTLScript && lineHasLTRScript { |
||||||
|
escaped.BadBIDI = true |
||||||
|
} |
||||||
|
lineHasBIDI = false |
||||||
|
lineHasRTLScript = false |
||||||
|
lineHasLTRScript = false |
||||||
|
|
||||||
|
case r == '\r' || r == '\t' || r == ' ': |
||||||
|
// These are acceptable control characters and space characters
|
||||||
|
case unicode.IsSpace(r): |
||||||
|
escaped.HasSpaces = true |
||||||
|
escaped.Escaped = true |
||||||
|
if writePos < i { |
||||||
|
if _, err = output.Write(bs[writePos:i]); err != nil { |
||||||
|
escaped.HasError = true |
||||||
|
return |
||||||
|
} |
||||||
|
} |
||||||
|
if err = writeEscaped(output, r); err != nil { |
||||||
|
escaped.HasError = true |
||||||
|
return |
||||||
|
} |
||||||
|
writePos = i + size |
||||||
|
case unicode.Is(unicode.Bidi_Control, r): |
||||||
|
escaped.Escaped = true |
||||||
|
escaped.HasBIDI = true |
||||||
|
if writePos < i { |
||||||
|
if _, err = output.Write(bs[writePos:i]); err != nil { |
||||||
|
escaped.HasError = true |
||||||
|
return |
||||||
|
} |
||||||
|
} |
||||||
|
lineHasBIDI = true |
||||||
|
if err = writeEscaped(output, r); err != nil { |
||||||
|
escaped.HasError = true |
||||||
|
return |
||||||
|
} |
||||||
|
writePos = i + size |
||||||
|
case unicode.Is(unicode.C, r): |
||||||
|
escaped.Escaped = true |
||||||
|
escaped.HasControls = true |
||||||
|
if writePos < i { |
||||||
|
if _, err = output.Write(bs[writePos:i]); err != nil { |
||||||
|
escaped.HasError = true |
||||||
|
return |
||||||
|
} |
||||||
|
} |
||||||
|
if err = writeEscaped(output, r); err != nil { |
||||||
|
escaped.HasError = true |
||||||
|
return |
||||||
|
} |
||||||
|
writePos = i + size |
||||||
|
case unicode.Is(unicode.M, r): |
||||||
|
escaped.Escaped = true |
||||||
|
escaped.HasMarks = true |
||||||
|
if writePos < i { |
||||||
|
if _, err = output.Write(bs[writePos:i]); err != nil { |
||||||
|
escaped.HasError = true |
||||||
|
return |
||||||
|
} |
||||||
|
} |
||||||
|
if err = writeEscaped(output, r); err != nil { |
||||||
|
escaped.HasError = true |
||||||
|
return |
||||||
|
} |
||||||
|
writePos = i + size |
||||||
|
default: |
||||||
|
p, _ := bidi.Lookup(bs[i : i+size]) |
||||||
|
c := p.Class() |
||||||
|
if c == bidi.R || c == bidi.AL { |
||||||
|
lineHasRTLScript = true |
||||||
|
escaped.HasRTLScript = true |
||||||
|
} else if c == bidi.L { |
||||||
|
lineHasLTRScript = true |
||||||
|
escaped.HasLTRScript = true |
||||||
|
} |
||||||
|
} |
||||||
|
i += size |
||||||
|
} |
||||||
|
if n > 0 { |
||||||
|
// we read something...
|
||||||
|
// write everything unwritten
|
||||||
|
if writePos < i { |
||||||
|
if _, err = output.Write(bs[writePos:i]); err != nil { |
||||||
|
escaped.HasError = true |
||||||
|
return |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
// reset the starting positions for the next read
|
||||||
|
readStart = 0 |
||||||
|
writePos = 0 |
||||||
|
} |
||||||
|
} |
||||||
|
if readStart > 0 { |
||||||
|
// this means that there is an incomplete or broken rune at 0-readStart and we read nothing on the last go round
|
||||||
|
escaped.Escaped = true |
||||||
|
escaped.HasBadRunes = true |
||||||
|
if err = writeBroken(output, buf[:readStart]); err != nil { |
||||||
|
escaped.HasError = true |
||||||
|
return |
||||||
|
} |
||||||
|
} |
||||||
|
if err == io.EOF { |
||||||
|
if lineHasBIDI && !lineHasRTLScript && lineHasLTRScript { |
||||||
|
escaped.BadBIDI = true |
||||||
|
} |
||||||
|
err = nil |
||||||
|
return |
||||||
|
} |
||||||
|
escaped.HasError = true |
||||||
|
return |
||||||
|
} |
||||||
|
|
||||||
|
func writeBroken(output io.Writer, bs []byte) (err error) { |
||||||
|
_, err = fmt.Fprintf(output, `<span class="broken-code-point"><%X></span>`, bs) |
||||||
|
return |
||||||
|
} |
||||||
|
|
||||||
|
func writeEscaped(output io.Writer, r rune) (err error) { |
||||||
|
_, err = fmt.Fprintf(output, `<span class="escaped-code-point" data-escaped="[U+%04X]"><span class="char">%c</span></span>`, r, r) |
||||||
|
return |
||||||
|
} |
||||||
@ -0,0 +1,17 @@ |
|||||||
|
{{if .EscapeStatus.BadBIDI}} |
||||||
|
<div class="ui error message unicode-escape-prompt"> |
||||||
|
<span class="close icon hide-panel button" data-panel-closest=".message">{{svg "octicon-x" 16 "close inside"}}</span> |
||||||
|
<div class="header"> |
||||||
|
{{$.root.i18n.Tr "repo.bidi_bad_header"}} |
||||||
|
</div> |
||||||
|
<p>{{$.root.i18n.Tr "repo.bidi_bad_description" | Str2html}}</p> |
||||||
|
</div> |
||||||
|
{{else if .EscapeStatus.Escaped}} |
||||||
|
<div class="ui warning message unicode-escape-prompt"> |
||||||
|
<span class="close icon hide-panel button" data-panel-closest=".message">{{svg "octicon-x" 16 "close inside"}}</span> |
||||||
|
<div class="header"> |
||||||
|
{{$.root.i18n.Tr "repo.unicode_header"}} |
||||||
|
</div> |
||||||
|
<p>{{$.root.i18n.Tr "repo.unicode_description" | Str2html}}</p> |
||||||
|
</div> |
||||||
|
{{end}} |
||||||
@ -0,0 +1,28 @@ |
|||||||
|
export function initUnicodeEscapeButton() { |
||||||
|
$(document).on('click', 'a.escape-button', (e) => { |
||||||
|
e.preventDefault(); |
||||||
|
$(e.target).parents('.file-content, .non-diff-file-content').find('.file-code, .file-view').addClass('unicode-escaped'); |
||||||
|
$(e.target).hide(); |
||||||
|
$(e.target).siblings('a.unescape-button').show(); |
||||||
|
}); |
||||||
|
$(document).on('click', 'a.unescape-button', (e) => { |
||||||
|
e.preventDefault(); |
||||||
|
$(e.target).parents('.file-content, .non-diff-file-content').find('.file-code, .file-view').removeClass('unicode-escaped'); |
||||||
|
$(e.target).hide(); |
||||||
|
$(e.target).siblings('a.escape-button').show(); |
||||||
|
}); |
||||||
|
$(document).on('click', 'a.toggle-escape-button', (e) => { |
||||||
|
e.preventDefault(); |
||||||
|
const fileContent = $(e.target).parents('.file-content, .non-diff-file-content'); |
||||||
|
const fileView = fileContent.find('.file-code, .file-view'); |
||||||
|
if (fileView.hasClass('unicode-escaped')) { |
||||||
|
fileView.removeClass('unicode-escaped'); |
||||||
|
fileContent.find('a.unescape-button').hide(); |
||||||
|
fileContent.find('a.escape-button').show(); |
||||||
|
} else { |
||||||
|
fileView.addClass('unicode-escaped'); |
||||||
|
fileContent.find('a.unescape-button').show(); |
||||||
|
fileContent.find('a.escape-button').hide(); |
||||||
|
} |
||||||
|
}); |
||||||
|
} |
||||||
Loading…
Reference in new issue