third_party
/
gitea
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix utf8 tests (#8192)

Browse Source 
* Prevent compiler environment from making the tests fail

* Remove unused function

* Pass lint
tokarchuk/v1.17
guillep2k 5 years ago committed by techknowlogick
parent 73f7e82024
commit 2628b15ee3
2 changed files with 81 additions and 35 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 2
      modules/charset/charset.go
  2. 114
      modules/charset/charset_test.go

2
modules/charset/charset.go
Unescape View File

@ -35,7 +35,7 @@ func ToUTF8WithErr(content []byte) (string, error) {
}
// If there is an error, we concatenate the nicely decoded part and the
// original left over. This way we won't lose data.
// original left over. This way we won't lose much data.
result, n, err := transform.Bytes(encoding.NewDecoder(), content)
if err != nil {
result = append(result, content[n:]...)

114
modules/charset/charset_test.go
Unescape View File

@ -24,41 +24,55 @@ func TestToUTF8WithErr(t *testing.T) {
var res string
var err error
// Note: golang compiler seems so behave differently depending on the current
// locale, so some conversions might behave differently. For that reason, we don't
// depend on particular conversions but in expected behaviors.
res, err = ToUTF8WithErr([]byte{0x41, 0x42, 0x43})
assert.Equal(t, "ABC", res)
assert.NoError(t, err)
assert.Equal(t, "ABC", res)
// "áéíóú"
res, err = ToUTF8WithErr([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba})
assert.Equal(t, "áéíóú", res)
assert.NoError(t, err)
assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, []byte(res))
res, err = ToUTF8WithErr([]byte{0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba})
assert.Equal(t, "áéíóú", res)
// "áéíóú"
res, err = ToUTF8WithErr([]byte{0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3,
0xc3, 0xba})
assert.NoError(t, err)
assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, []byte(res))
// This test FAILS
res, err = ToUTF8WithErr([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0xF1, 0x6F, 0x73})
assert.Equal(t, "Hola, así cómo ños", res)
res, err = ToUTF8WithErr([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63,
0xF3, 0x6D, 0x6F, 0x20, 0xF1, 0x6F, 0x73, 0x41, 0x41, 0x41, 0x2e})
assert.NoError(t, err)
stringMustStartWith(t, "Hola,", res)
stringMustEndWith(t, "AAA.", res)
res, err = ToUTF8WithErr([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x07, 0xA4, 0x6F, 0x73})
// Do not fail for differences in invalid cases, as the library might change the conversion criteria for those
assert.Regexp(t, "^Hola, así cómo", res)
res, err = ToUTF8WithErr([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63,
0xF3, 0x6D, 0x6F, 0x20, 0x07, 0xA4, 0x6F, 0x73, 0x41, 0x41, 0x41, 0x2e})
assert.NoError(t, err)
stringMustStartWith(t, "Hola,", res)
stringMustEndWith(t, "AAA.", res)
res, err = ToUTF8WithErr([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x81, 0xA4, 0x6F, 0x73})
// Do not fail for differences in invalid cases, as the library might change the conversion criteria for those
assert.Regexp(t, "^Hola, así cómo", res)
res, err = ToUTF8WithErr([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63,
0xF3, 0x6D, 0x6F, 0x20, 0x81, 0xA4, 0x6F, 0x73, 0x41, 0x41, 0x41, 0x2e})
assert.NoError(t, err)
stringMustStartWith(t, "Hola,", res)
stringMustEndWith(t, "AAA.", res)
// Japanese (Shift-JIS)
res, err = ToUTF8WithErr([]byte{0x93, 0xFA, 0x91, 0xAE, 0x94, 0xE9, 0x82, 0xBC, 0x82, 0xB5, 0x82, 0xBF, 0x82, 0xE3, 0x81, 0x42})
assert.Equal(t, "日属秘ぞしちゅ。", res)
// 日属秘ぞしちゅ。
res, err = ToUTF8WithErr([]byte{0x93, 0xFA, 0x91, 0xAE, 0x94, 0xE9, 0x82, 0xBC, 0x82, 0xB5, 0x82,
0xBF, 0x82, 0xE3, 0x81, 0x42})
assert.NoError(t, err)
assert.Equal(t, []byte{0xE6, 0x97, 0xA5, 0xE5, 0xB1, 0x9E, 0xE7, 0xA7, 0x98, 0xE3,
0x81, 0x9E, 0xE3, 0x81, 0x97, 0xE3, 0x81, 0xA1, 0xE3, 0x82, 0x85, 0xE3, 0x80, 0x82},
[]byte(res))
res, err = ToUTF8WithErr([]byte{0x00, 0x00, 0x00, 0x00})
assert.Equal(t, "\x00\x00\x00\x00", res)
assert.NoError(t, err)
assert.Equal(t, []byte{0x00, 0x00, 0x00, 0x00}, []byte(res))
}
func TestToUTF8WithFallback(t *testing.T) {
@ -75,8 +89,10 @@ func TestToUTF8WithFallback(t *testing.T) {
assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res)
// "Hola, así cómo ños"
res = ToUTF8WithFallback([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0xF1, 0x6F, 0x73})
assert.Equal(t, []byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xC3, 0xAD, 0x20, 0x63, 0xC3, 0xB3, 0x6D, 0x6F, 0x20, 0xC3, 0xB1, 0x6F, 0x73}, res)
res = ToUTF8WithFallback([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63,
0xF3, 0x6D, 0x6F, 0x20, 0xF1, 0x6F, 0x73})
assert.Equal(t, []byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xC3, 0xAD, 0x20, 0x63,
0xC3, 0xB3, 0x6D, 0x6F, 0x20, 0xC3, 0xB1, 0x6F, 0x73}, res)
// "Hola, así cómo "
minmatch := []byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xC3, 0xAD, 0x20, 0x63, 0xC3, 0xB3, 0x6D, 0x6F, 0x20}
@ -100,22 +116,36 @@ func TestToUTF8WithFallback(t *testing.T) {
}
func TestToUTF8(t *testing.T) {
res := ToUTF8("ABC")
assert.Equal(t, "ABC", res)
res = ToUTF8("áéíóú")
assert.Equal(t, "áéíóú", res)
// Note: golang compiler seems so behave differently depending on the current
// locale, so some conversions might behave differently. For that reason, we don't
// depend on particular conversions but in expected behaviors.
// With utf-8 BOM
res = ToUTF8("\ufeffáéíóú")
assert.Equal(t, "áéíóú", res)
res = ToUTF8("Hola, así cómo ños")
assert.Equal(t, "Hola, así cómo ños", res)
res := ToUTF8(string([]byte{0x41, 0x42, 0x43}))
assert.Equal(t, "ABC", res)
res = ToUTF8("Hola, así cómo \x07ños")
// Do not fail for differences in invalid cases, as the library might change the conversion criteria for those
assert.Regexp(t, "^Hola, así cómo", res)
// "áéíóú"
res = ToUTF8(string([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}))
assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, []byte(res))
// BOM + "áéíóú"
res = ToUTF8(string([]byte{0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3,
0xc3, 0xba}))
assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, []byte(res))
// Latin1
// Hola, así cómo ños
res = ToUTF8(string([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63,
0xF3, 0x6D, 0x6F, 0x20, 0xF1, 0x6F, 0x73}))
assert.Equal(t, []byte{0x48, 0x6f, 0x6c, 0x61, 0x2c, 0x20, 0x61, 0x73, 0xc3, 0xad, 0x20, 0x63,
0xc3, 0xb3, 0x6d, 0x6f, 0x20, 0xc3, 0xb1, 0x6f, 0x73}, []byte(res))
// Latin1
// Hola, así cómo \x07ños
res = ToUTF8(string([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63,
0xF3, 0x6D, 0x6F, 0x20, 0x07, 0xA4, 0x6F, 0x73}))
// Hola,
bytesMustStartWith(t, []byte{0x48, 0x6F, 0x6C, 0x61, 0x2C}, []byte(res))
// This test FAILS
// res = ToUTF8("Hola, así cómo \x81ños")
@ -123,11 +153,15 @@ func TestToUTF8(t *testing.T) {
// assert.Regexp(t, "^Hola, así cómo", res)
// Japanese (Shift-JIS)
res = ToUTF8("\x93\xFA\x91\xAE\x94\xE9\x82\xBC\x82\xB5\x82\xBF\x82\xE3\x81\x42")
assert.Equal(t, "日属秘ぞしちゅ。", res)
// 日属秘ぞしちゅ。
res = ToUTF8(string([]byte{0x93, 0xFA, 0x91, 0xAE, 0x94, 0xE9, 0x82, 0xBC, 0x82, 0xB5, 0x82,
0xBF, 0x82, 0xE3, 0x81, 0x42}))
assert.Equal(t, []byte{0xE6, 0x97, 0xA5, 0xE5, 0xB1, 0x9E, 0xE7, 0xA7, 0x98, 0xE3,
0x81, 0x9E, 0xE3, 0x81, 0x97, 0xE3, 0x81, 0xA1, 0xE3, 0x82, 0x85, 0xE3, 0x80, 0x82},
[]byte(res))
res = ToUTF8("\x00\x00\x00\x00")
assert.Equal(t, "\x00\x00\x00\x00", res)
assert.Equal(t, []byte{0x00, 0x00, 0x00, 0x00}, []byte(res))
}
func TestToUTF8DropErrors(t *testing.T) {
@ -203,3 +237,15 @@ func TestDetectEncoding(t *testing.T) {
_, err = DetectEncoding(b)
assert.Error(t, err)
}
func stringMustStartWith(t *testing.T, expected string, value string) {
assert.Equal(t, expected, string(value[:len(expected)]))
}
func stringMustEndWith(t *testing.T, expected string, value string) {
assert.Equal(t, expected, string(value[len(value)-len(expected):]))
}
func bytesMustStartWith(t *testing.T, expected []byte, value []byte) {
assert.Equal(t, expected, value[:len(expected)])
}

Write Preview
Loading…
Cancel
Save