Update to last common x/text (#3994)
Antoine GIRARD
7 years ago
committed by
Lunny Xiao
parent
000b2d33a9
commit
edc78b9633
@ -1,524 +0,0 @@ |
||||
// Copyright 2013 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// +build ignore
|
||||
|
||||
package main |
||||
|
||||
import ( |
||||
"bufio" |
||||
"fmt" |
||||
"log" |
||||
"net/http" |
||||
"sort" |
||||
"strings" |
||||
"unicode/utf8" |
||||
|
||||
"golang.org/x/text/encoding" |
||||
"golang.org/x/text/internal/gen" |
||||
) |
||||
|
||||
const ascii = "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" + |
||||
"\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f" + |
||||
` !"#$%&'()*+,-./0123456789:;<=>?` + |
||||
`@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_` + |
||||
"`abcdefghijklmnopqrstuvwxyz{|}~\u007f" |
||||
|
||||
var encodings = []struct { |
||||
name string |
||||
mib string |
||||
comment string |
||||
varName string |
||||
replacement byte |
||||
mapping string |
||||
}{ |
||||
{ |
||||
"IBM Code Page 437", |
||||
"PC8CodePage437", |
||||
"", |
||||
"CodePage437", |
||||
encoding.ASCIISub, |
||||
"http://source.icu-project.org/repos/icu/data/trunk/charset/data/ucm/glibc-IBM437-2.1.2.ucm", |
||||
}, |
||||
{ |
||||
"IBM Code Page 850", |
||||
"PC850Multilingual", |
||||
"", |
||||
"CodePage850", |
||||
encoding.ASCIISub, |
||||
"http://source.icu-project.org/repos/icu/data/trunk/charset/data/ucm/glibc-IBM850-2.1.2.ucm", |
||||
}, |
||||
{ |
||||
"IBM Code Page 852", |
||||
"PCp852", |
||||
"", |
||||
"CodePage852", |
||||
encoding.ASCIISub, |
||||
"http://source.icu-project.org/repos/icu/data/trunk/charset/data/ucm/glibc-IBM852-2.1.2.ucm", |
||||
}, |
||||
{ |
||||
"IBM Code Page 855", |
||||
"IBM855", |
||||
"", |
||||
"CodePage855", |
||||
encoding.ASCIISub, |
||||
"http://source.icu-project.org/repos/icu/data/trunk/charset/data/ucm/glibc-IBM855-2.1.2.ucm", |
||||
}, |
||||
{ |
||||
"Windows Code Page 858", // PC latin1 with Euro
|
||||
"IBM00858", |
||||
"", |
||||
"CodePage858", |
||||
encoding.ASCIISub, |
||||
"http://source.icu-project.org/repos/icu/data/trunk/charset/data/ucm/windows-858-2000.ucm", |
||||
}, |
||||
{ |
||||
"IBM Code Page 860", |
||||
"IBM860", |
||||
"", |
||||
"CodePage860", |
||||
encoding.ASCIISub, |
||||
"http://source.icu-project.org/repos/icu/data/trunk/charset/data/ucm/glibc-IBM860-2.1.2.ucm", |
||||
}, |
||||
{ |
||||
"IBM Code Page 862", |
||||
"PC862LatinHebrew", |
||||
"", |
||||
"CodePage862", |
||||
encoding.ASCIISub, |
||||
"http://source.icu-project.org/repos/icu/data/trunk/charset/data/ucm/glibc-IBM862-2.1.2.ucm", |
||||
}, |
||||
{ |
||||
"IBM Code Page 863", |
||||
"IBM863", |
||||
"", |
||||
"CodePage863", |
||||
encoding.ASCIISub, |
||||
"http://source.icu-project.org/repos/icu/data/trunk/charset/data/ucm/glibc-IBM863-2.1.2.ucm", |
||||
}, |
||||
{ |
||||
"IBM Code Page 865", |
||||
"IBM865", |
||||
"", |
||||
"CodePage865", |
||||
encoding.ASCIISub, |
||||
"http://source.icu-project.org/repos/icu/data/trunk/charset/data/ucm/glibc-IBM865-2.1.2.ucm", |
||||
}, |
||||
{ |
||||
"IBM Code Page 866", |
||||
"IBM866", |
||||
"", |
||||
"CodePage866", |
||||
encoding.ASCIISub, |
||||
"http://encoding.spec.whatwg.org/index-ibm866.txt", |
||||
}, |
||||
{ |
||||
"ISO 8859-1", |
||||
"ISOLatin1", |
||||
"", |
||||
"ISO8859_1", |
||||
encoding.ASCIISub, |
||||
"http://source.icu-project.org/repos/icu/data/trunk/charset/data/ucm/iso-8859_1-1998.ucm", |
||||
}, |
||||
{ |
||||
"ISO 8859-2", |
||||
"ISOLatin2", |
||||
"", |
||||
"ISO8859_2", |
||||
encoding.ASCIISub, |
||||
"http://encoding.spec.whatwg.org/index-iso-8859-2.txt", |
||||
}, |
||||
{ |
||||
"ISO 8859-3", |
||||
"ISOLatin3", |
||||
"", |
||||
"ISO8859_3", |
||||
encoding.ASCIISub, |
||||
"http://encoding.spec.whatwg.org/index-iso-8859-3.txt", |
||||
}, |
||||
{ |
||||
"ISO 8859-4", |
||||
"ISOLatin4", |
||||
"", |
||||
"ISO8859_4", |
||||
encoding.ASCIISub, |
||||
"http://encoding.spec.whatwg.org/index-iso-8859-4.txt", |
||||
}, |
||||
{ |
||||
"ISO 8859-5", |
||||
"ISOLatinCyrillic", |
||||
"", |
||||
"ISO8859_5", |
||||
encoding.ASCIISub, |
||||
"http://encoding.spec.whatwg.org/index-iso-8859-5.txt", |
||||
}, |
||||
{ |
||||
"ISO 8859-6", |
||||
"ISOLatinArabic", |
||||
"", |
||||
"ISO8859_6,ISO8859_6E,ISO8859_6I", |
||||
encoding.ASCIISub, |
||||
"http://encoding.spec.whatwg.org/index-iso-8859-6.txt", |
||||
}, |
||||
{ |
||||
"ISO 8859-7", |
||||
"ISOLatinGreek", |
||||
"", |
||||
"ISO8859_7", |
||||
encoding.ASCIISub, |
||||
"http://encoding.spec.whatwg.org/index-iso-8859-7.txt", |
||||
}, |
||||
{ |
||||
"ISO 8859-8", |
||||
"ISOLatinHebrew", |
||||
"", |
||||
"ISO8859_8,ISO8859_8E,ISO8859_8I", |
||||
encoding.ASCIISub, |
||||
"http://encoding.spec.whatwg.org/index-iso-8859-8.txt", |
||||
}, |
||||
{ |
||||
"ISO 8859-10", |
||||
"ISOLatin6", |
||||
"", |
||||
"ISO8859_10", |
||||
encoding.ASCIISub, |
||||
"http://encoding.spec.whatwg.org/index-iso-8859-10.txt", |
||||
}, |
||||
{ |
||||
"ISO 8859-13", |
||||
"ISO885913", |
||||
"", |
||||
"ISO8859_13", |
||||
encoding.ASCIISub, |
||||
"http://encoding.spec.whatwg.org/index-iso-8859-13.txt", |
||||
}, |
||||
{ |
||||
"ISO 8859-14", |
||||
"ISO885914", |
||||
"", |
||||
"ISO8859_14", |
||||
encoding.ASCIISub, |
||||
"http://encoding.spec.whatwg.org/index-iso-8859-14.txt", |
||||
}, |
||||
{ |
||||
"ISO 8859-15", |
||||
"ISO885915", |
||||
"", |
||||
"ISO8859_15", |
||||
encoding.ASCIISub, |
||||
"http://encoding.spec.whatwg.org/index-iso-8859-15.txt", |
||||
}, |
||||
{ |
||||
"ISO 8859-16", |
||||
"ISO885916", |
||||
"", |
||||
"ISO8859_16", |
||||
encoding.ASCIISub, |
||||
"http://encoding.spec.whatwg.org/index-iso-8859-16.txt", |
||||
}, |
||||
{ |
||||
"KOI8-R", |
||||
"KOI8R", |
||||
"", |
||||
"KOI8R", |
||||
encoding.ASCIISub, |
||||
"http://encoding.spec.whatwg.org/index-koi8-r.txt", |
||||
}, |
||||
{ |
||||
"KOI8-U", |
||||
"KOI8U", |
||||
"", |
||||
"KOI8U", |
||||
encoding.ASCIISub, |
||||
"http://encoding.spec.whatwg.org/index-koi8-u.txt", |
||||
}, |
||||
{ |
||||
"Macintosh", |
||||
"Macintosh", |
||||
"", |
||||
"Macintosh", |
||||
encoding.ASCIISub, |
||||
"http://encoding.spec.whatwg.org/index-macintosh.txt", |
||||
}, |
||||
{ |
||||
"Macintosh Cyrillic", |
||||
"MacintoshCyrillic", |
||||
"", |
||||
"MacintoshCyrillic", |
||||
encoding.ASCIISub, |
||||
"http://encoding.spec.whatwg.org/index-x-mac-cyrillic.txt", |
||||
}, |
||||
{ |
||||
"Windows 874", |
||||
"Windows874", |
||||
"", |
||||
"Windows874", |
||||
encoding.ASCIISub, |
||||
"http://encoding.spec.whatwg.org/index-windows-874.txt", |
||||
}, |
||||
{ |
||||
"Windows 1250", |
||||
"Windows1250", |
||||
"", |
||||
"Windows1250", |
||||
encoding.ASCIISub, |
||||
"http://encoding.spec.whatwg.org/index-windows-1250.txt", |
||||
}, |
||||
{ |
||||
"Windows 1251", |
||||
"Windows1251", |
||||
"", |
||||
"Windows1251", |
||||
encoding.ASCIISub, |
||||
"http://encoding.spec.whatwg.org/index-windows-1251.txt", |
||||
}, |
||||
{ |
||||
"Windows 1252", |
||||
"Windows1252", |
||||
"", |
||||
"Windows1252", |
||||
encoding.ASCIISub, |
||||
"http://encoding.spec.whatwg.org/index-windows-1252.txt", |
||||
}, |
||||
{ |
||||
"Windows 1253", |
||||
"Windows1253", |
||||
"", |
||||
"Windows1253", |
||||
encoding.ASCIISub, |
||||
"http://encoding.spec.whatwg.org/index-windows-1253.txt", |
||||
}, |
||||
{ |
||||
"Windows 1254", |
||||
"Windows1254", |
||||
"", |
||||
"Windows1254", |
||||
encoding.ASCIISub, |
||||
"http://encoding.spec.whatwg.org/index-windows-1254.txt", |
||||
}, |
||||
{ |
||||
"Windows 1255", |
||||
"Windows1255", |
||||
"", |
||||
"Windows1255", |
||||
encoding.ASCIISub, |
||||
"http://encoding.spec.whatwg.org/index-windows-1255.txt", |
||||
}, |
||||
{ |
||||
"Windows 1256", |
||||
"Windows1256", |
||||
"", |
||||
"Windows1256", |
||||
encoding.ASCIISub, |
||||
"http://encoding.spec.whatwg.org/index-windows-1256.txt", |
||||
}, |
||||
{ |
||||
"Windows 1257", |
||||
"Windows1257", |
||||
"", |
||||
"Windows1257", |
||||
encoding.ASCIISub, |
||||
"http://encoding.spec.whatwg.org/index-windows-1257.txt", |
||||
}, |
||||
{ |
||||
"Windows 1258", |
||||
"Windows1258", |
||||
"", |
||||
"Windows1258", |
||||
encoding.ASCIISub, |
||||
"http://encoding.spec.whatwg.org/index-windows-1258.txt", |
||||
}, |
||||
{ |
||||
"X-User-Defined", |
||||
"XUserDefined", |
||||
"It is defined at http://encoding.spec.whatwg.org/#x-user-defined", |
||||
"XUserDefined", |
||||
encoding.ASCIISub, |
||||
ascii + |
||||
"\uf780\uf781\uf782\uf783\uf784\uf785\uf786\uf787" + |
||||
"\uf788\uf789\uf78a\uf78b\uf78c\uf78d\uf78e\uf78f" + |
||||
"\uf790\uf791\uf792\uf793\uf794\uf795\uf796\uf797" + |
||||
"\uf798\uf799\uf79a\uf79b\uf79c\uf79d\uf79e\uf79f" + |
||||
"\uf7a0\uf7a1\uf7a2\uf7a3\uf7a4\uf7a5\uf7a6\uf7a7" + |
||||
"\uf7a8\uf7a9\uf7aa\uf7ab\uf7ac\uf7ad\uf7ae\uf7af" + |
||||
"\uf7b0\uf7b1\uf7b2\uf7b3\uf7b4\uf7b5\uf7b6\uf7b7" + |
||||
"\uf7b8\uf7b9\uf7ba\uf7bb\uf7bc\uf7bd\uf7be\uf7bf" + |
||||
"\uf7c0\uf7c1\uf7c2\uf7c3\uf7c4\uf7c5\uf7c6\uf7c7" + |
||||
"\uf7c8\uf7c9\uf7ca\uf7cb\uf7cc\uf7cd\uf7ce\uf7cf" + |
||||
"\uf7d0\uf7d1\uf7d2\uf7d3\uf7d4\uf7d5\uf7d6\uf7d7" + |
||||
"\uf7d8\uf7d9\uf7da\uf7db\uf7dc\uf7dd\uf7de\uf7df" + |
||||
"\uf7e0\uf7e1\uf7e2\uf7e3\uf7e4\uf7e5\uf7e6\uf7e7" + |
||||
"\uf7e8\uf7e9\uf7ea\uf7eb\uf7ec\uf7ed\uf7ee\uf7ef" + |
||||
"\uf7f0\uf7f1\uf7f2\uf7f3\uf7f4\uf7f5\uf7f6\uf7f7" + |
||||
"\uf7f8\uf7f9\uf7fa\uf7fb\uf7fc\uf7fd\uf7fe\uf7ff", |
||||
}, |
||||
} |
||||
|
||||
func getWHATWG(url string) string { |
||||
res, err := http.Get(url) |
||||
if err != nil { |
||||
log.Fatalf("%q: Get: %v", url, err) |
||||
} |
||||
defer res.Body.Close() |
||||
|
||||
mapping := make([]rune, 128) |
||||
for i := range mapping { |
||||
mapping[i] = '\ufffd' |
||||
} |
||||
|
||||
scanner := bufio.NewScanner(res.Body) |
||||
for scanner.Scan() { |
||||
s := strings.TrimSpace(scanner.Text()) |
||||
if s == "" || s[0] == '#' { |
||||
continue |
||||
} |
||||
x, y := 0, 0 |
||||
if _, err := fmt.Sscanf(s, "%d\t0x%x", &x, &y); err != nil { |
||||
log.Fatalf("could not parse %q", s) |
||||
} |
||||
if x < 0 || 128 <= x { |
||||
log.Fatalf("code %d is out of range", x) |
||||
} |
||||
if 0x80 <= y && y < 0xa0 { |
||||
// We diverge from the WHATWG spec by mapping control characters
|
||||
// in the range [0x80, 0xa0) to U+FFFD.
|
||||
continue |
||||
} |
||||
mapping[x] = rune(y) |
||||
} |
||||
return ascii + string(mapping) |
||||
} |
||||
|
||||
func getUCM(url string) string { |
||||
res, err := http.Get(url) |
||||
if err != nil { |
||||
log.Fatalf("%q: Get: %v", url, err) |
||||
} |
||||
defer res.Body.Close() |
||||
|
||||
mapping := make([]rune, 256) |
||||
for i := range mapping { |
||||
mapping[i] = '\ufffd' |
||||
} |
||||
|
||||
charsFound := 0 |
||||
scanner := bufio.NewScanner(res.Body) |
||||
for scanner.Scan() { |
||||
s := strings.TrimSpace(scanner.Text()) |
||||
if s == "" || s[0] == '#' { |
||||
continue |
||||
} |
||||
var c byte |
||||
var r rune |
||||
if _, err := fmt.Sscanf(s, `<U%x> \x%x |0`, &r, &c); err != nil { |
||||
continue |
||||
} |
||||
mapping[c] = r |
||||
charsFound++ |
||||
} |
||||
|
||||
if charsFound < 200 { |
||||
log.Fatalf("%q: only %d characters found (wrong page format?)", url, charsFound) |
||||
} |
||||
|
||||
return string(mapping) |
||||
} |
||||
|
||||
func main() { |
||||
mibs := map[string]bool{} |
||||
all := []string{} |
||||
|
||||
w := gen.NewCodeWriter() |
||||
defer w.WriteGoFile("tables.go", "charmap") |
||||
|
||||
printf := func(s string, a ...interface{}) { fmt.Fprintf(w, s, a...) } |
||||
|
||||
printf("import (\n") |
||||
printf("\t\"golang.org/x/text/encoding\"\n") |
||||
printf("\t\"golang.org/x/text/encoding/internal/identifier\"\n") |
||||
printf(")\n\n") |
||||
for _, e := range encodings { |
||||
varNames := strings.Split(e.varName, ",") |
||||
all = append(all, varNames...) |
||||
varName := varNames[0] |
||||
switch { |
||||
case strings.HasPrefix(e.mapping, "http://encoding.spec.whatwg.org/"): |
||||
e.mapping = getWHATWG(e.mapping) |
||||
case strings.HasPrefix(e.mapping, "http://source.icu-project.org/repos/icu/data/trunk/charset/data/ucm/"): |
||||
e.mapping = getUCM(e.mapping) |
||||
} |
||||
|
||||
asciiSuperset, low := strings.HasPrefix(e.mapping, ascii), 0x00 |
||||
if asciiSuperset { |
||||
low = 0x80 |
||||
} |
||||
lvn := 1 |
||||
if strings.HasPrefix(varName, "ISO") || strings.HasPrefix(varName, "KOI") { |
||||
lvn = 3 |
||||
} |
||||
lowerVarName := strings.ToLower(varName[:lvn]) + varName[lvn:] |
||||
printf("// %s is the %s encoding.\n", varName, e.name) |
||||
if e.comment != "" { |
||||
printf("//\n// %s\n", e.comment) |
||||
} |
||||
printf("var %s encoding.Encoding = &%s\n\nvar %s = charmap{\nname: %q,\n", |
||||
varName, lowerVarName, lowerVarName, e.name) |
||||
if mibs[e.mib] { |
||||
log.Fatalf("MIB type %q declared multiple times.", e.mib) |
||||
} |
||||
printf("mib: identifier.%s,\n", e.mib) |
||||
printf("asciiSuperset: %t,\n", asciiSuperset) |
||||
printf("low: 0x%02x,\n", low) |
||||
printf("replacement: 0x%02x,\n", e.replacement) |
||||
|
||||
printf("decode: [256]utf8Enc{\n") |
||||
i, backMapping := 0, map[rune]byte{} |
||||
for _, c := range e.mapping { |
||||
if _, ok := backMapping[c]; !ok && c != utf8.RuneError { |
||||
backMapping[c] = byte(i) |
||||
} |
||||
var buf [8]byte |
||||
n := utf8.EncodeRune(buf[:], c) |
||||
if n > 3 { |
||||
panic(fmt.Sprintf("rune %q (%U) is too long", c, c)) |
||||
} |
||||
printf("{%d,[3]byte{0x%02x,0x%02x,0x%02x}},", n, buf[0], buf[1], buf[2]) |
||||
if i%2 == 1 { |
||||
printf("\n") |
||||
} |
||||
i++ |
||||
} |
||||
printf("},\n") |
||||
|
||||
printf("encode: [256]uint32{\n") |
||||
encode := make([]uint32, 0, 256) |
||||
for c, i := range backMapping { |
||||
encode = append(encode, uint32(i)<<24|uint32(c)) |
||||
} |
||||
sort.Sort(byRune(encode)) |
||||
for len(encode) < cap(encode) { |
||||
encode = append(encode, encode[len(encode)-1]) |
||||
} |
||||
for i, enc := range encode { |
||||
printf("0x%08x,", enc) |
||||
if i%8 == 7 { |
||||
printf("\n") |
||||
} |
||||
} |
||||
printf("},\n}\n") |
||||
|
||||
// Add an estimate of the size of a single charmap{} struct value, which
|
||||
// includes two 256 elem arrays of 4 bytes and some extra fields, which
|
||||
// align to 3 uint64s on 64-bit architectures.
|
||||
w.Size += 2*4*256 + 3*8 |
||||
} |
||||
// TODO: add proper line breaking.
|
||||
printf("var listAll = []encoding.Encoding{\n%s,\n}\n\n", strings.Join(all, ",\n")) |
||||
} |
||||
|
||||
type byRune []uint32 |
||||
|
||||
func (b byRune) Len() int { return len(b) } |
||||
func (b byRune) Less(i, j int) bool { return b[i]&0xffffff < b[j]&0xffffff } |
||||
func (b byRune) Swap(i, j int) { b[i], b[j] = b[j], b[i] } |
||||
File diff suppressed because it is too large
Load Diff
@ -1,167 +0,0 @@ |
||||
// Copyright 2015 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// +build ignore
|
||||
|
||||
package main |
||||
|
||||
import ( |
||||
"bytes" |
||||
"encoding/json" |
||||
"fmt" |
||||
"log" |
||||
"strings" |
||||
|
||||
"golang.org/x/text/internal/gen" |
||||
) |
||||
|
||||
type group struct { |
||||
Encodings []struct { |
||||
Labels []string |
||||
Name string |
||||
} |
||||
} |
||||
|
||||
func main() { |
||||
gen.Init() |
||||
|
||||
r := gen.Open("http://www.w3.org/TR", "w3", "encoding/indexes/encodings.json") |
||||
var groups []group |
||||
if err := json.NewDecoder(r).Decode(&groups); err != nil { |
||||
log.Fatalf("Error reading encodings.json: %v", err) |
||||
} |
||||
|
||||
w := &bytes.Buffer{} |
||||
fmt.Fprintln(w, "type htmlEncoding byte") |
||||
fmt.Fprintln(w, "const (") |
||||
for i, g := range groups { |
||||
for _, e := range g.Encodings { |
||||
name := consts[e.Name] |
||||
if name == "" { |
||||
log.Fatalf("No const defined for %s.", e.Name) |
||||
} |
||||
if i == 0 { |
||||
fmt.Fprintf(w, "%s htmlEncoding = iota\n", name) |
||||
} else { |
||||
fmt.Fprintf(w, "%s\n", name) |
||||
} |
||||
} |
||||
} |
||||
fmt.Fprintln(w, "numEncodings") |
||||
fmt.Fprint(w, ")\n\n") |
||||
|
||||
fmt.Fprintln(w, "var canonical = [numEncodings]string{") |
||||
for _, g := range groups { |
||||
for _, e := range g.Encodings { |
||||
fmt.Fprintf(w, "%q,\n", e.Name) |
||||
} |
||||
} |
||||
fmt.Fprint(w, "}\n\n") |
||||
|
||||
fmt.Fprintln(w, "var nameMap = map[string]htmlEncoding{") |
||||
for _, g := range groups { |
||||
for _, e := range g.Encodings { |
||||
for _, l := range e.Labels { |
||||
fmt.Fprintf(w, "%q: %s,\n", l, consts[e.Name]) |
||||
} |
||||
} |
||||
} |
||||
fmt.Fprint(w, "}\n\n") |
||||
|
||||
var tags []string |
||||
fmt.Fprintln(w, "var localeMap = []htmlEncoding{") |
||||
for _, loc := range locales { |
||||
tags = append(tags, loc.tag) |
||||
fmt.Fprintf(w, "%s, // %s \n", consts[loc.name], loc.tag) |
||||
} |
||||
fmt.Fprint(w, "}\n\n") |
||||
|
||||
fmt.Fprintf(w, "const locales = %q\n", strings.Join(tags, " ")) |
||||
|
||||
gen.WriteGoFile("tables.go", "htmlindex", w.Bytes()) |
||||
} |
||||
|
||||
// consts maps canonical encoding name to internal constant.
|
||||
var consts = map[string]string{ |
||||
"utf-8": "utf8", |
||||
"ibm866": "ibm866", |
||||
"iso-8859-2": "iso8859_2", |
||||
"iso-8859-3": "iso8859_3", |
||||
"iso-8859-4": "iso8859_4", |
||||
"iso-8859-5": "iso8859_5", |
||||
"iso-8859-6": "iso8859_6", |
||||
"iso-8859-7": "iso8859_7", |
||||
"iso-8859-8": "iso8859_8", |
||||
"iso-8859-8-i": "iso8859_8I", |
||||
"iso-8859-10": "iso8859_10", |
||||
"iso-8859-13": "iso8859_13", |
||||
"iso-8859-14": "iso8859_14", |
||||
"iso-8859-15": "iso8859_15", |
||||
"iso-8859-16": "iso8859_16", |
||||
"koi8-r": "koi8r", |
||||
"koi8-u": "koi8u", |
||||
"macintosh": "macintosh", |
||||
"windows-874": "windows874", |
||||
"windows-1250": "windows1250", |
||||
"windows-1251": "windows1251", |
||||
"windows-1252": "windows1252", |
||||
"windows-1253": "windows1253", |
||||
"windows-1254": "windows1254", |
||||
"windows-1255": "windows1255", |
||||
"windows-1256": "windows1256", |
||||
"windows-1257": "windows1257", |
||||
"windows-1258": "windows1258", |
||||
"x-mac-cyrillic": "macintoshCyrillic", |
||||
"gbk": "gbk", |
||||
"gb18030": "gb18030", |
||||
// "hz-gb-2312": "hzgb2312", // Was removed from WhatWG
|
||||
"big5": "big5", |
||||
"euc-jp": "eucjp", |
||||
"iso-2022-jp": "iso2022jp", |
||||
"shift_jis": "shiftJIS", |
||||
"euc-kr": "euckr", |
||||
"replacement": "replacement", |
||||
"utf-16be": "utf16be", |
||||
"utf-16le": "utf16le", |
||||
"x-user-defined": "xUserDefined", |
||||
} |
||||
|
||||
// locales is taken from
|
||||
// https://html.spec.whatwg.org/multipage/syntax.html#encoding-sniffing-algorithm.
|
||||
var locales = []struct{ tag, name string }{ |
||||
{"und", "windows-1252"}, // The default value.
|
||||
{"ar", "windows-1256"}, |
||||
{"ba", "windows-1251"}, |
||||
{"be", "windows-1251"}, |
||||
{"bg", "windows-1251"}, |
||||
{"cs", "windows-1250"}, |
||||
{"el", "iso-8859-7"}, |
||||
{"et", "windows-1257"}, |
||||
{"fa", "windows-1256"}, |
||||
{"he", "windows-1255"}, |
||||
{"hr", "windows-1250"}, |
||||
{"hu", "iso-8859-2"}, |
||||
{"ja", "shift_jis"}, |
||||
{"kk", "windows-1251"}, |
||||
{"ko", "euc-kr"}, |
||||
{"ku", "windows-1254"}, |
||||
{"ky", "windows-1251"}, |
||||
{"lt", "windows-1257"}, |
||||
{"lv", "windows-1257"}, |
||||
{"mk", "windows-1251"}, |
||||
{"pl", "iso-8859-2"}, |
||||
{"ru", "windows-1251"}, |
||||
{"sah", "windows-1251"}, |
||||
{"sk", "windows-1250"}, |
||||
{"sl", "iso-8859-2"}, |
||||
{"sr", "windows-1251"}, |
||||
{"tg", "windows-1251"}, |
||||
{"th", "windows-874"}, |
||||
{"tr", "windows-1254"}, |
||||
{"tt", "windows-1251"}, |
||||
{"uk", "windows-1251"}, |
||||
{"vi", "windows-1258"}, |
||||
{"zh-hans", "gb18030"}, |
||||
{"zh-hant", "big5"}, |
||||
} |
||||
@ -1,137 +0,0 @@ |
||||
// Copyright 2015 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// +build ignore
|
||||
|
||||
package main |
||||
|
||||
import ( |
||||
"bytes" |
||||
"encoding/xml" |
||||
"fmt" |
||||
"io" |
||||
"log" |
||||
"strings" |
||||
|
||||
"golang.org/x/text/internal/gen" |
||||
) |
||||
|
||||
type registry struct { |
||||
XMLName xml.Name `xml:"registry"` |
||||
Updated string `xml:"updated"` |
||||
Registry []struct { |
||||
ID string `xml:"id,attr"` |
||||
Record []struct { |
||||
Name string `xml:"name"` |
||||
Xref []struct { |
||||
Type string `xml:"type,attr"` |
||||
Data string `xml:"data,attr"` |
||||
} `xml:"xref"` |
||||
Desc struct { |
||||
Data string `xml:",innerxml"` |
||||
// Any []struct {
|
||||
// Data string `xml:",chardata"`
|
||||
// } `xml:",any"`
|
||||
// Data string `xml:",chardata"`
|
||||
} `xml:"description,"` |
||||
MIB string `xml:"value"` |
||||
Alias []string `xml:"alias"` |
||||
MIME string `xml:"preferred_alias"` |
||||
} `xml:"record"` |
||||
} `xml:"registry"` |
||||
} |
||||
|
||||
func main() { |
||||
r := gen.OpenIANAFile("assignments/character-sets/character-sets.xml") |
||||
reg := ®istry{} |
||||
if err := xml.NewDecoder(r).Decode(®); err != nil && err != io.EOF { |
||||
log.Fatalf("Error decoding charset registry: %v", err) |
||||
} |
||||
if len(reg.Registry) == 0 || reg.Registry[0].ID != "character-sets-1" { |
||||
log.Fatalf("Unexpected ID %s", reg.Registry[0].ID) |
||||
} |
||||
|
||||
w := &bytes.Buffer{} |
||||
fmt.Fprintf(w, "const (\n") |
||||
for _, rec := range reg.Registry[0].Record { |
||||
constName := "" |
||||
for _, a := range rec.Alias { |
||||
if strings.HasPrefix(a, "cs") && strings.IndexByte(a, '-') == -1 { |
||||
// Some of the constant definitions have comments in them. Strip those.
|
||||
constName = strings.Title(strings.SplitN(a[2:], "\n", 2)[0]) |
||||
} |
||||
} |
||||
if constName == "" { |
||||
switch rec.MIB { |
||||
case "2085": |
||||
constName = "HZGB2312" // Not listed as alias for some reason.
|
||||
default: |
||||
log.Fatalf("No cs alias defined for %s.", rec.MIB) |
||||
} |
||||
} |
||||
if rec.MIME != "" { |
||||
rec.MIME = fmt.Sprintf(" (MIME: %s)", rec.MIME) |
||||
} |
||||
fmt.Fprintf(w, "// %s is the MIB identifier with IANA name %s%s.\n//\n", constName, rec.Name, rec.MIME) |
||||
if len(rec.Desc.Data) > 0 { |
||||
fmt.Fprint(w, "// ") |
||||
d := xml.NewDecoder(strings.NewReader(rec.Desc.Data)) |
||||
inElem := true |
||||
attr := "" |
||||
for { |
||||
t, err := d.Token() |
||||
if err != nil { |
||||
if err != io.EOF { |
||||
log.Fatal(err) |
||||
} |
||||
break |
||||
} |
||||
switch x := t.(type) { |
||||
case xml.CharData: |
||||
attr = "" // Don't need attribute info.
|
||||
a := bytes.Split([]byte(x), []byte("\n")) |
||||
for i, b := range a { |
||||
if b = bytes.TrimSpace(b); len(b) != 0 { |
||||
if !inElem && i > 0 { |
||||
fmt.Fprint(w, "\n// ") |
||||
} |
||||
inElem = false |
||||
fmt.Fprintf(w, "%s ", string(b)) |
||||
} |
||||
} |
||||
case xml.StartElement: |
||||
if x.Name.Local == "xref" { |
||||
inElem = true |
||||
use := false |
||||
for _, a := range x.Attr { |
||||
if a.Name.Local == "type" { |
||||
use = use || a.Value != "person" |
||||
} |
||||
if a.Name.Local == "data" && use { |
||||
attr = a.Value + " " |
||||
} |
||||
} |
||||
} |
||||
case xml.EndElement: |
||||
inElem = false |
||||
fmt.Fprint(w, attr) |
||||
} |
||||
} |
||||
fmt.Fprint(w, "\n") |
||||
} |
||||
for _, x := range rec.Xref { |
||||
switch x.Type { |
||||
case "rfc": |
||||
fmt.Fprintf(w, "// Reference: %s\n", strings.ToUpper(x.Data)) |
||||
case "uri": |
||||
fmt.Fprintf(w, "// Reference: %s\n", x.Data) |
||||
} |
||||
} |
||||
fmt.Fprintf(w, "%s MIB = %s\n", constName, rec.MIB) |
||||
fmt.Fprintln(w) |
||||
} |
||||
fmt.Fprintln(w, ")") |
||||
|
||||
gen.WriteGoFile("mib.go", "identifier", w.Bytes()) |
||||
} |
||||
@ -1,161 +0,0 @@ |
||||
// Copyright 2013 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// +build ignore
|
||||
|
||||
package main |
||||
|
||||
// This program generates tables.go:
|
||||
// go run maketables.go | gofmt > tables.go
|
||||
|
||||
// TODO: Emoji extensions?
|
||||
// http://www.unicode.org/faq/emoji_dingbats.html
|
||||
// http://www.unicode.org/Public/UNIDATA/EmojiSources.txt
|
||||
|
||||
import ( |
||||
"bufio" |
||||
"fmt" |
||||
"log" |
||||
"net/http" |
||||
"sort" |
||||
"strings" |
||||
) |
||||
|
||||
type entry struct { |
||||
jisCode, table int |
||||
} |
||||
|
||||
func main() { |
||||
fmt.Printf("// generated by go run maketables.go; DO NOT EDIT\n\n") |
||||
fmt.Printf("// Package japanese provides Japanese encodings such as EUC-JP and Shift JIS.\n") |
||||
fmt.Printf(`package japanese // import "golang.org/x/text/encoding/japanese"` + "\n\n") |
||||
|
||||
reverse := [65536]entry{} |
||||
for i := range reverse { |
||||
reverse[i].table = -1 |
||||
} |
||||
|
||||
tables := []struct { |
||||
url string |
||||
name string |
||||
}{ |
||||
{"http://encoding.spec.whatwg.org/index-jis0208.txt", "0208"}, |
||||
{"http://encoding.spec.whatwg.org/index-jis0212.txt", "0212"}, |
||||
} |
||||
for i, table := range tables { |
||||
res, err := http.Get(table.url) |
||||
if err != nil { |
||||
log.Fatalf("%q: Get: %v", table.url, err) |
||||
} |
||||
defer res.Body.Close() |
||||
|
||||
mapping := [65536]uint16{} |
||||
|
||||
scanner := bufio.NewScanner(res.Body) |
||||
for scanner.Scan() { |
||||
s := strings.TrimSpace(scanner.Text()) |
||||
if s == "" || s[0] == '#' { |
||||
continue |
||||
} |
||||
x, y := 0, uint16(0) |
||||
if _, err := fmt.Sscanf(s, "%d 0x%x", &x, &y); err != nil { |
||||
log.Fatalf("%q: could not parse %q", table.url, s) |
||||
} |
||||
if x < 0 || 120*94 <= x { |
||||
log.Fatalf("%q: JIS code %d is out of range", table.url, x) |
||||
} |
||||
mapping[x] = y |
||||
if reverse[y].table == -1 { |
||||
reverse[y] = entry{jisCode: x, table: i} |
||||
} |
||||
} |
||||
if err := scanner.Err(); err != nil { |
||||
log.Fatalf("%q: scanner error: %v", table.url, err) |
||||
} |
||||
|
||||
fmt.Printf("// jis%sDecode is the decoding table from JIS %s code to Unicode.\n// It is defined at %s\n", |
||||
table.name, table.name, table.url) |
||||
fmt.Printf("var jis%sDecode = [...]uint16{\n", table.name) |
||||
for i, m := range mapping { |
||||
if m != 0 { |
||||
fmt.Printf("\t%d: 0x%04X,\n", i, m) |
||||
} |
||||
} |
||||
fmt.Printf("}\n\n") |
||||
} |
||||
|
||||
// Any run of at least separation continuous zero entries in the reverse map will
|
||||
// be a separate encode table.
|
||||
const separation = 1024 |
||||
|
||||
intervals := []interval(nil) |
||||
low, high := -1, -1 |
||||
for i, v := range reverse { |
||||
if v.table == -1 { |
||||
continue |
||||
} |
||||
if low < 0 { |
||||
low = i |
||||
} else if i-high >= separation { |
||||
if high >= 0 { |
||||
intervals = append(intervals, interval{low, high}) |
||||
} |
||||
low = i |
||||
} |
||||
high = i + 1 |
||||
} |
||||
if high >= 0 { |
||||
intervals = append(intervals, interval{low, high}) |
||||
} |
||||
sort.Sort(byDecreasingLength(intervals)) |
||||
|
||||
fmt.Printf("const (\n") |
||||
fmt.Printf("\tjis0208 = 1\n") |
||||
fmt.Printf("\tjis0212 = 2\n") |
||||
fmt.Printf("\tcodeMask = 0x7f\n") |
||||
fmt.Printf("\tcodeShift = 7\n") |
||||
fmt.Printf("\ttableShift = 14\n") |
||||
fmt.Printf(")\n\n") |
||||
|
||||
fmt.Printf("const numEncodeTables = %d\n\n", len(intervals)) |
||||
fmt.Printf("// encodeX are the encoding tables from Unicode to JIS code,\n") |
||||
fmt.Printf("// sorted by decreasing length.\n") |
||||
for i, v := range intervals { |
||||
fmt.Printf("// encode%d: %5d entries for runes in [%5d, %5d).\n", i, v.len(), v.low, v.high) |
||||
} |
||||
fmt.Printf("//\n") |
||||
fmt.Printf("// The high two bits of the value record whether the JIS code comes from the\n") |
||||
fmt.Printf("// JIS0208 table (high bits == 1) or the JIS0212 table (high bits == 2).\n") |
||||
fmt.Printf("// The low 14 bits are two 7-bit unsigned integers j1 and j2 that form the\n") |
||||
fmt.Printf("// JIS code (94*j1 + j2) within that table.\n") |
||||
fmt.Printf("\n") |
||||
|
||||
for i, v := range intervals { |
||||
fmt.Printf("const encode%dLow, encode%dHigh = %d, %d\n\n", i, i, v.low, v.high) |
||||
fmt.Printf("var encode%d = [...]uint16{\n", i) |
||||
for j := v.low; j < v.high; j++ { |
||||
x := reverse[j] |
||||
if x.table == -1 { |
||||
continue |
||||
} |
||||
fmt.Printf("\t%d - %d: jis%s<<14 | 0x%02X<<7 | 0x%02X,\n", |
||||
j, v.low, tables[x.table].name, x.jisCode/94, x.jisCode%94) |
||||
} |
||||
fmt.Printf("}\n\n") |
||||
} |
||||
} |
||||
|
||||
// interval is a half-open interval [low, high).
|
||||
type interval struct { |
||||
low, high int |
||||
} |
||||
|
||||
func (i interval) len() int { return i.high - i.low } |
||||
|
||||
// byDecreasingLength sorts intervals by decreasing length.
|
||||
type byDecreasingLength []interval |
||||
|
||||
func (b byDecreasingLength) Len() int { return len(b) } |
||||
func (b byDecreasingLength) Less(i, j int) bool { return b[i].len() > b[j].len() } |
||||
func (b byDecreasingLength) Swap(i, j int) { b[i], b[j] = b[j], b[i] } |
||||
@ -1,143 +0,0 @@ |
||||
// Copyright 2013 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// +build ignore
|
||||
|
||||
package main |
||||
|
||||
// This program generates tables.go:
|
||||
// go run maketables.go | gofmt > tables.go
|
||||
|
||||
import ( |
||||
"bufio" |
||||
"fmt" |
||||
"log" |
||||
"net/http" |
||||
"sort" |
||||
"strings" |
||||
) |
||||
|
||||
func main() { |
||||
fmt.Printf("// generated by go run maketables.go; DO NOT EDIT\n\n") |
||||
fmt.Printf("// Package korean provides Korean encodings such as EUC-KR.\n") |
||||
fmt.Printf(`package korean // import "golang.org/x/text/encoding/korean"` + "\n\n") |
||||
|
||||
res, err := http.Get("http://encoding.spec.whatwg.org/index-euc-kr.txt") |
||||
if err != nil { |
||||
log.Fatalf("Get: %v", err) |
||||
} |
||||
defer res.Body.Close() |
||||
|
||||
mapping := [65536]uint16{} |
||||
reverse := [65536]uint16{} |
||||
|
||||
scanner := bufio.NewScanner(res.Body) |
||||
for scanner.Scan() { |
||||
s := strings.TrimSpace(scanner.Text()) |
||||
if s == "" || s[0] == '#' { |
||||
continue |
||||
} |
||||
x, y := uint16(0), uint16(0) |
||||
if _, err := fmt.Sscanf(s, "%d 0x%x", &x, &y); err != nil { |
||||
log.Fatalf("could not parse %q", s) |
||||
} |
||||
if x < 0 || 178*(0xc7-0x81)+(0xfe-0xc7)*94+(0xff-0xa1) <= x { |
||||
log.Fatalf("EUC-KR code %d is out of range", x) |
||||
} |
||||
mapping[x] = y |
||||
if reverse[y] == 0 { |
||||
c0, c1 := uint16(0), uint16(0) |
||||
if x < 178*(0xc7-0x81) { |
||||
c0 = uint16(x/178) + 0x81 |
||||
c1 = uint16(x % 178) |
||||
switch { |
||||
case c1 < 1*26: |
||||
c1 += 0x41 |
||||
case c1 < 2*26: |
||||
c1 += 0x47 |
||||
default: |
||||
c1 += 0x4d |
||||
} |
||||
} else { |
||||
x -= 178 * (0xc7 - 0x81) |
||||
c0 = uint16(x/94) + 0xc7 |
||||
c1 = uint16(x%94) + 0xa1 |
||||
} |
||||
reverse[y] = c0<<8 | c1 |
||||
} |
||||
} |
||||
if err := scanner.Err(); err != nil { |
||||
log.Fatalf("scanner error: %v", err) |
||||
} |
||||
|
||||
fmt.Printf("// decode is the decoding table from EUC-KR code to Unicode.\n") |
||||
fmt.Printf("// It is defined at http://encoding.spec.whatwg.org/index-euc-kr.txt\n") |
||||
fmt.Printf("var decode = [...]uint16{\n") |
||||
for i, v := range mapping { |
||||
if v != 0 { |
||||
fmt.Printf("\t%d: 0x%04X,\n", i, v) |
||||
} |
||||
} |
||||
fmt.Printf("}\n\n") |
||||
|
||||
// Any run of at least separation continuous zero entries in the reverse map will
|
||||
// be a separate encode table.
|
||||
const separation = 1024 |
||||
|
||||
intervals := []interval(nil) |
||||
low, high := -1, -1 |
||||
for i, v := range reverse { |
||||
if v == 0 { |
||||
continue |
||||
} |
||||
if low < 0 { |
||||
low = i |
||||
} else if i-high >= separation { |
||||
if high >= 0 { |
||||
intervals = append(intervals, interval{low, high}) |
||||
} |
||||
low = i |
||||
} |
||||
high = i + 1 |
||||
} |
||||
if high >= 0 { |
||||
intervals = append(intervals, interval{low, high}) |
||||
} |
||||
sort.Sort(byDecreasingLength(intervals)) |
||||
|
||||
fmt.Printf("const numEncodeTables = %d\n\n", len(intervals)) |
||||
fmt.Printf("// encodeX are the encoding tables from Unicode to EUC-KR code,\n") |
||||
fmt.Printf("// sorted by decreasing length.\n") |
||||
for i, v := range intervals { |
||||
fmt.Printf("// encode%d: %5d entries for runes in [%5d, %5d).\n", i, v.len(), v.low, v.high) |
||||
} |
||||
fmt.Printf("\n") |
||||
|
||||
for i, v := range intervals { |
||||
fmt.Printf("const encode%dLow, encode%dHigh = %d, %d\n\n", i, i, v.low, v.high) |
||||
fmt.Printf("var encode%d = [...]uint16{\n", i) |
||||
for j := v.low; j < v.high; j++ { |
||||
x := reverse[j] |
||||
if x == 0 { |
||||
continue |
||||
} |
||||
fmt.Printf("\t%d-%d: 0x%04X,\n", j, v.low, x) |
||||
} |
||||
fmt.Printf("}\n\n") |
||||
} |
||||
} |
||||
|
||||
// interval is a half-open interval [low, high).
|
||||
type interval struct { |
||||
low, high int |
||||
} |
||||
|
||||
func (i interval) len() int { return i.high - i.low } |
||||
|
||||
// byDecreasingLength sorts intervals by decreasing length.
|
||||
type byDecreasingLength []interval |
||||
|
||||
func (b byDecreasingLength) Len() int { return len(b) } |
||||
func (b byDecreasingLength) Less(i, j int) bool { return b[i].len() > b[j].len() } |
||||
func (b byDecreasingLength) Swap(i, j int) { b[i], b[j] = b[j], b[i] } |
||||
@ -1,161 +0,0 @@ |
||||
// Copyright 2013 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// +build ignore
|
||||
|
||||
package main |
||||
|
||||
// This program generates tables.go:
|
||||
// go run maketables.go | gofmt > tables.go
|
||||
|
||||
import ( |
||||
"bufio" |
||||
"fmt" |
||||
"log" |
||||
"net/http" |
||||
"sort" |
||||
"strings" |
||||
) |
||||
|
||||
func main() { |
||||
fmt.Printf("// generated by go run maketables.go; DO NOT EDIT\n\n") |
||||
fmt.Printf("// Package simplifiedchinese provides Simplified Chinese encodings such as GBK.\n") |
||||
fmt.Printf(`package simplifiedchinese // import "golang.org/x/text/encoding/simplifiedchinese"` + "\n\n") |
||||
|
||||
printGB18030() |
||||
printGBK() |
||||
} |
||||
|
||||
func printGB18030() { |
||||
res, err := http.Get("http://encoding.spec.whatwg.org/index-gb18030.txt") |
||||
if err != nil { |
||||
log.Fatalf("Get: %v", err) |
||||
} |
||||
defer res.Body.Close() |
||||
|
||||
fmt.Printf("// gb18030 is the table from http://encoding.spec.whatwg.org/index-gb18030.txt\n") |
||||
fmt.Printf("var gb18030 = [...][2]uint16{\n") |
||||
scanner := bufio.NewScanner(res.Body) |
||||
for scanner.Scan() { |
||||
s := strings.TrimSpace(scanner.Text()) |
||||
if s == "" || s[0] == '#' { |
||||
continue |
||||
} |
||||
x, y := uint32(0), uint32(0) |
||||
if _, err := fmt.Sscanf(s, "%d 0x%x", &x, &y); err != nil { |
||||
log.Fatalf("could not parse %q", s) |
||||
} |
||||
if x < 0x10000 && y < 0x10000 { |
||||
fmt.Printf("\t{0x%04x, 0x%04x},\n", x, y) |
||||
} |
||||
} |
||||
fmt.Printf("}\n\n") |
||||
} |
||||
|
||||
func printGBK() { |
||||
res, err := http.Get("http://encoding.spec.whatwg.org/index-gbk.txt") |
||||
if err != nil { |
||||
log.Fatalf("Get: %v", err) |
||||
} |
||||
defer res.Body.Close() |
||||
|
||||
mapping := [65536]uint16{} |
||||
reverse := [65536]uint16{} |
||||
|
||||
scanner := bufio.NewScanner(res.Body) |
||||
for scanner.Scan() { |
||||
s := strings.TrimSpace(scanner.Text()) |
||||
if s == "" || s[0] == '#' { |
||||
continue |
||||
} |
||||
x, y := uint16(0), uint16(0) |
||||
if _, err := fmt.Sscanf(s, "%d 0x%x", &x, &y); err != nil { |
||||
log.Fatalf("could not parse %q", s) |
||||
} |
||||
if x < 0 || 126*190 <= x { |
||||
log.Fatalf("GBK code %d is out of range", x) |
||||
} |
||||
mapping[x] = y |
||||
if reverse[y] == 0 { |
||||
c0, c1 := x/190, x%190 |
||||
if c1 >= 0x3f { |
||||
c1++ |
||||
} |
||||
reverse[y] = (0x81+c0)<<8 | (0x40 + c1) |
||||
} |
||||
} |
||||
if err := scanner.Err(); err != nil { |
||||
log.Fatalf("scanner error: %v", err) |
||||
} |
||||
|
||||
fmt.Printf("// decode is the decoding table from GBK code to Unicode.\n") |
||||
fmt.Printf("// It is defined at http://encoding.spec.whatwg.org/index-gbk.txt\n") |
||||
fmt.Printf("var decode = [...]uint16{\n") |
||||
for i, v := range mapping { |
||||
if v != 0 { |
||||
fmt.Printf("\t%d: 0x%04X,\n", i, v) |
||||
} |
||||
} |
||||
fmt.Printf("}\n\n") |
||||
|
||||
// Any run of at least separation continuous zero entries in the reverse map will
|
||||
// be a separate encode table.
|
||||
const separation = 1024 |
||||
|
||||
intervals := []interval(nil) |
||||
low, high := -1, -1 |
||||
for i, v := range reverse { |
||||
if v == 0 { |
||||
continue |
||||
} |
||||
if low < 0 { |
||||
low = i |
||||
} else if i-high >= separation { |
||||
if high >= 0 { |
||||
intervals = append(intervals, interval{low, high}) |
||||
} |
||||
low = i |
||||
} |
||||
high = i + 1 |
||||
} |
||||
if high >= 0 { |
||||
intervals = append(intervals, interval{low, high}) |
||||
} |
||||
sort.Sort(byDecreasingLength(intervals)) |
||||
|
||||
fmt.Printf("const numEncodeTables = %d\n\n", len(intervals)) |
||||
fmt.Printf("// encodeX are the encoding tables from Unicode to GBK code,\n") |
||||
fmt.Printf("// sorted by decreasing length.\n") |
||||
for i, v := range intervals { |
||||
fmt.Printf("// encode%d: %5d entries for runes in [%5d, %5d).\n", i, v.len(), v.low, v.high) |
||||
} |
||||
fmt.Printf("\n") |
||||
|
||||
for i, v := range intervals { |
||||
fmt.Printf("const encode%dLow, encode%dHigh = %d, %d\n\n", i, i, v.low, v.high) |
||||
fmt.Printf("var encode%d = [...]uint16{\n", i) |
||||
for j := v.low; j < v.high; j++ { |
||||
x := reverse[j] |
||||
if x == 0 { |
||||
continue |
||||
} |
||||
fmt.Printf("\t%d-%d: 0x%04X,\n", j, v.low, x) |
||||
} |
||||
fmt.Printf("}\n\n") |
||||
} |
||||
} |
||||
|
||||
// interval is a half-open interval [low, high).
|
||||
type interval struct { |
||||
low, high int |
||||
} |
||||
|
||||
func (i interval) len() int { return i.high - i.low } |
||||
|
||||
// byDecreasingLength sorts intervals by decreasing length.
|
||||
type byDecreasingLength []interval |
||||
|
||||
func (b byDecreasingLength) Len() int { return len(b) } |
||||
func (b byDecreasingLength) Less(i, j int) bool { return b[i].len() > b[j].len() } |
||||
func (b byDecreasingLength) Swap(i, j int) { b[i], b[j] = b[j], b[i] } |
||||
@ -1,140 +0,0 @@ |
||||
// Copyright 2013 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// +build ignore
|
||||
|
||||
package main |
||||
|
||||
// This program generates tables.go:
|
||||
// go run maketables.go | gofmt > tables.go
|
||||
|
||||
import ( |
||||
"bufio" |
||||
"fmt" |
||||
"log" |
||||
"net/http" |
||||
"sort" |
||||
"strings" |
||||
) |
||||
|
||||
func main() { |
||||
fmt.Printf("// generated by go run maketables.go; DO NOT EDIT\n\n") |
||||
fmt.Printf("// Package traditionalchinese provides Traditional Chinese encodings such as Big5.\n") |
||||
fmt.Printf(`package traditionalchinese // import "golang.org/x/text/encoding/traditionalchinese"` + "\n\n") |
||||
|
||||
res, err := http.Get("http://encoding.spec.whatwg.org/index-big5.txt") |
||||
if err != nil { |
||||
log.Fatalf("Get: %v", err) |
||||
} |
||||
defer res.Body.Close() |
||||
|
||||
mapping := [65536]uint32{} |
||||
reverse := [65536 * 4]uint16{} |
||||
|
||||
scanner := bufio.NewScanner(res.Body) |
||||
for scanner.Scan() { |
||||
s := strings.TrimSpace(scanner.Text()) |
||||
if s == "" || s[0] == '#' { |
||||
continue |
||||
} |
||||
x, y := uint16(0), uint32(0) |
||||
if _, err := fmt.Sscanf(s, "%d 0x%x", &x, &y); err != nil { |
||||
log.Fatalf("could not parse %q", s) |
||||
} |
||||
if x < 0 || 126*157 <= x { |
||||
log.Fatalf("Big5 code %d is out of range", x) |
||||
} |
||||
mapping[x] = y |
||||
|
||||
// The WHATWG spec http://encoding.spec.whatwg.org/#indexes says that
|
||||
// "The index pointer for code point in index is the first pointer
|
||||
// corresponding to code point in index", which would normally mean
|
||||
// that the code below should be guarded by "if reverse[y] == 0", but
|
||||
// last instead of first seems to match the behavior of
|
||||
// "iconv -f UTF-8 -t BIG5". For example, U+8005 者 occurs twice in
|
||||
// http://encoding.spec.whatwg.org/index-big5.txt, as index 2148
|
||||
// (encoded as "\x8e\xcd") and index 6543 (encoded as "\xaa\xcc")
|
||||
// and "echo 者 | iconv -f UTF-8 -t BIG5 | xxd" gives "\xaa\xcc".
|
||||
c0, c1 := x/157, x%157 |
||||
if c1 < 0x3f { |
||||
c1 += 0x40 |
||||
} else { |
||||
c1 += 0x62 |
||||
} |
||||
reverse[y] = (0x81+c0)<<8 | c1 |
||||
} |
||||
if err := scanner.Err(); err != nil { |
||||
log.Fatalf("scanner error: %v", err) |
||||
} |
||||
|
||||
fmt.Printf("// decode is the decoding table from Big5 code to Unicode.\n") |
||||
fmt.Printf("// It is defined at http://encoding.spec.whatwg.org/index-big5.txt\n") |
||||
fmt.Printf("var decode = [...]uint32{\n") |
||||
for i, v := range mapping { |
||||
if v != 0 { |
||||
fmt.Printf("\t%d: 0x%08X,\n", i, v) |
||||
} |
||||
} |
||||
fmt.Printf("}\n\n") |
||||
|
||||
// Any run of at least separation continuous zero entries in the reverse map will
|
||||
// be a separate encode table.
|
||||
const separation = 1024 |
||||
|
||||
intervals := []interval(nil) |
||||
low, high := -1, -1 |
||||
for i, v := range reverse { |
||||
if v == 0 { |
||||
continue |
||||
} |
||||
if low < 0 { |
||||
low = i |
||||
} else if i-high >= separation { |
||||
if high >= 0 { |
||||
intervals = append(intervals, interval{low, high}) |
||||
} |
||||
low = i |
||||
} |
||||
high = i + 1 |
||||
} |
||||
if high >= 0 { |
||||
intervals = append(intervals, interval{low, high}) |
||||
} |
||||
sort.Sort(byDecreasingLength(intervals)) |
||||
|
||||
fmt.Printf("const numEncodeTables = %d\n\n", len(intervals)) |
||||
fmt.Printf("// encodeX are the encoding tables from Unicode to Big5 code,\n") |
||||
fmt.Printf("// sorted by decreasing length.\n") |
||||
for i, v := range intervals { |
||||
fmt.Printf("// encode%d: %5d entries for runes in [%6d, %6d).\n", i, v.len(), v.low, v.high) |
||||
} |
||||
fmt.Printf("\n") |
||||
|
||||
for i, v := range intervals { |
||||
fmt.Printf("const encode%dLow, encode%dHigh = %d, %d\n\n", i, i, v.low, v.high) |
||||
fmt.Printf("var encode%d = [...]uint16{\n", i) |
||||
for j := v.low; j < v.high; j++ { |
||||
x := reverse[j] |
||||
if x == 0 { |
||||
continue |
||||
} |
||||
fmt.Printf("\t%d-%d: 0x%04X,\n", j, v.low, x) |
||||
} |
||||
fmt.Printf("}\n\n") |
||||
} |
||||
} |
||||
|
||||
// interval is a half-open interval [low, high).
|
||||
type interval struct { |
||||
low, high int |
||||
} |
||||
|
||||
func (i interval) len() int { return i.high - i.low } |
||||
|
||||
// byDecreasingLength sorts intervals by decreasing length.
|
||||
type byDecreasingLength []interval |
||||
|
||||
func (b byDecreasingLength) Len() int { return len(b) } |
||||
func (b byDecreasingLength) Less(i, j int) bool { return b[i].len() > b[j].len() } |
||||
func (b byDecreasingLength) Swap(i, j int) { b[i], b[j] = b[j], b[i] } |
||||
@ -1,20 +0,0 @@ |
||||
// Copyright 2014 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// +build ignore
|
||||
|
||||
package main |
||||
|
||||
// This file contains code common to the maketables.go and the package code.
|
||||
|
||||
// langAliasType is the type of an alias in langAliasMap.
|
||||
type langAliasType int8 |
||||
|
||||
const ( |
||||
langDeprecated langAliasType = iota |
||||
langMacro |
||||
langLegacy |
||||
|
||||
langAliasTypeUnknown langAliasType = -1 |
||||
) |
||||
@ -1,162 +0,0 @@ |
||||
// Copyright 2015 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// +build ignore
|
||||
|
||||
package main |
||||
|
||||
// This file generates derivative tables based on the language package itself.
|
||||
|
||||
import ( |
||||
"bytes" |
||||
"flag" |
||||
"fmt" |
||||
"io/ioutil" |
||||
"log" |
||||
"reflect" |
||||
"sort" |
||||
"strings" |
||||
|
||||
"golang.org/x/text/internal/gen" |
||||
"golang.org/x/text/language" |
||||
"golang.org/x/text/unicode/cldr" |
||||
) |
||||
|
||||
var ( |
||||
test = flag.Bool("test", false, |
||||
"test existing tables; can be used to compare web data with package data.") |
||||
|
||||
draft = flag.String("draft", |
||||
"contributed", |
||||
`Minimal draft requirements (approved, contributed, provisional, unconfirmed).`) |
||||
) |
||||
|
||||
func main() { |
||||
gen.Init() |
||||
|
||||
// Read the CLDR zip file.
|
||||
r := gen.OpenCLDRCoreZip() |
||||
defer r.Close() |
||||
|
||||
d := &cldr.Decoder{} |
||||
data, err := d.DecodeZip(r) |
||||
if err != nil { |
||||
log.Fatalf("DecodeZip: %v", err) |
||||
} |
||||
|
||||
w := gen.NewCodeWriter() |
||||
defer func() { |
||||
buf := &bytes.Buffer{} |
||||
|
||||
if _, err = w.WriteGo(buf, "language"); err != nil { |
||||
log.Fatalf("Error formatting file index.go: %v", err) |
||||
} |
||||
|
||||
// Since we're generating a table for our own package we need to rewrite
|
||||
// doing the equivalent of go fmt -r 'language.b -> b'. Using
|
||||
// bytes.Replace will do.
|
||||
out := bytes.Replace(buf.Bytes(), []byte("language."), nil, -1) |
||||
if err := ioutil.WriteFile("index.go", out, 0600); err != nil { |
||||
log.Fatalf("Could not create file index.go: %v", err) |
||||
} |
||||
}() |
||||
|
||||
m := map[language.Tag]bool{} |
||||
for _, lang := range data.Locales() { |
||||
// We include all locales unconditionally to be consistent with en_US.
|
||||
// We want en_US, even though it has no data associated with it.
|
||||
|
||||
// TODO: put any of the languages for which no data exists at the end
|
||||
// of the index. This allows all components based on ICU to use that
|
||||
// as the cutoff point.
|
||||
// if x := data.RawLDML(lang); false ||
|
||||
// x.LocaleDisplayNames != nil ||
|
||||
// x.Characters != nil ||
|
||||
// x.Delimiters != nil ||
|
||||
// x.Measurement != nil ||
|
||||
// x.Dates != nil ||
|
||||
// x.Numbers != nil ||
|
||||
// x.Units != nil ||
|
||||
// x.ListPatterns != nil ||
|
||||
// x.Collations != nil ||
|
||||
// x.Segmentations != nil ||
|
||||
// x.Rbnf != nil ||
|
||||
// x.Annotations != nil ||
|
||||
// x.Metadata != nil {
|
||||
|
||||
// TODO: support POSIX natively, albeit non-standard.
|
||||
tag := language.Make(strings.Replace(lang, "_POSIX", "-u-va-posix", 1)) |
||||
m[tag] = true |
||||
// }
|
||||
} |
||||
// Include locales for plural rules, which uses a different structure.
|
||||
for _, plurals := range data.Supplemental().Plurals { |
||||
for _, rules := range plurals.PluralRules { |
||||
for _, lang := range strings.Split(rules.Locales, " ") { |
||||
m[language.Make(lang)] = true |
||||
} |
||||
} |
||||
} |
||||
|
||||
var core, special []language.Tag |
||||
|
||||
for t := range m { |
||||
if x := t.Extensions(); len(x) != 0 && fmt.Sprint(x) != "[u-va-posix]" { |
||||
log.Fatalf("Unexpected extension %v in %v", x, t) |
||||
} |
||||
if len(t.Variants()) == 0 && len(t.Extensions()) == 0 { |
||||
core = append(core, t) |
||||
} else { |
||||
special = append(special, t) |
||||
} |
||||
} |
||||
|
||||
w.WriteComment(` |
||||
NumCompactTags is the number of common tags. The maximum tag is |
||||
NumCompactTags-1.`) |
||||
w.WriteConst("NumCompactTags", len(core)+len(special)) |
||||
|
||||
sort.Sort(byAlpha(special)) |
||||
w.WriteVar("specialTags", special) |
||||
|
||||
// TODO: order by frequency?
|
||||
sort.Sort(byAlpha(core)) |
||||
|
||||
// Size computations are just an estimate.
|
||||
w.Size += int(reflect.TypeOf(map[uint32]uint16{}).Size()) |
||||
w.Size += len(core) * 6 // size of uint32 and uint16
|
||||
|
||||
fmt.Fprintln(w) |
||||
fmt.Fprintln(w, "var coreTags = map[uint32]uint16{") |
||||
fmt.Fprintln(w, "0x0: 0, // und") |
||||
i := len(special) + 1 // Und and special tags already written.
|
||||
for _, t := range core { |
||||
if t == language.Und { |
||||
continue |
||||
} |
||||
fmt.Fprint(w.Hash, t, i) |
||||
b, s, r := t.Raw() |
||||
fmt.Fprintf(w, "0x%s%s%s: %d, // %s\n", |
||||
getIndex(b, 3), // 3 is enough as it is guaranteed to be a compact number
|
||||
getIndex(s, 2), |
||||
getIndex(r, 3), |
||||
i, t) |
||||
i++ |
||||
} |
||||
fmt.Fprintln(w, "}") |
||||
} |
||||
|
||||
// getIndex prints the subtag type and extracts its index of size nibble.
|
||||
// If the index is less than n nibbles, the result is prefixed with 0s.
|
||||
func getIndex(x interface{}, n int) string { |
||||
s := fmt.Sprintf("%#v", x) // s is of form Type{typeID: 0x00}
|
||||
s = s[strings.Index(s, "0x")+2 : len(s)-1] |
||||
return strings.Repeat("0", n-len(s)) + s |
||||
} |
||||
|
||||
type byAlpha []language.Tag |
||||
|
||||
func (a byAlpha) Len() int { return len(a) } |
||||
func (a byAlpha) Swap(i, j int) { a[i], a[j] = a[j], a[i] } |
||||
func (a byAlpha) Less(i, j int) bool { return a[i].String() < a[j].String() } |
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@ -1,976 +0,0 @@ |
||||
// Copyright 2011 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// +build ignore
|
||||
|
||||
// Normalization table generator.
|
||||
// Data read from the web.
|
||||
// See forminfo.go for a description of the trie values associated with each rune.
|
||||
|
||||
package main |
||||
|
||||
import ( |
||||
"bytes" |
||||
"flag" |
||||
"fmt" |
||||
"io" |
||||
"log" |
||||
"sort" |
||||
"strconv" |
||||
"strings" |
||||
|
||||
"golang.org/x/text/internal/gen" |
||||
"golang.org/x/text/internal/triegen" |
||||
"golang.org/x/text/internal/ucd" |
||||
) |
||||
|
||||
func main() { |
||||
gen.Init() |
||||
loadUnicodeData() |
||||
compactCCC() |
||||
loadCompositionExclusions() |
||||
completeCharFields(FCanonical) |
||||
completeCharFields(FCompatibility) |
||||
computeNonStarterCounts() |
||||
verifyComputed() |
||||
printChars() |
||||
testDerived() |
||||
printTestdata() |
||||
makeTables() |
||||
} |
||||
|
||||
var ( |
||||
tablelist = flag.String("tables", |
||||
"all", |
||||
"comma-separated list of which tables to generate; "+ |
||||
"can be 'decomp', 'recomp', 'info' and 'all'") |
||||
test = flag.Bool("test", |
||||
false, |
||||
"test existing tables against DerivedNormalizationProps and generate test data for regression testing") |
||||
verbose = flag.Bool("verbose", |
||||
false, |
||||
"write data to stdout as it is parsed") |
||||
) |
||||
|
||||
const MaxChar = 0x10FFFF // anything above this shouldn't exist
|
||||
|
||||
// Quick Check properties of runes allow us to quickly
|
||||
// determine whether a rune may occur in a normal form.
|
||||
// For a given normal form, a rune may be guaranteed to occur
|
||||
// verbatim (QC=Yes), may or may not combine with another
|
||||
// rune (QC=Maybe), or may not occur (QC=No).
|
||||
type QCResult int |
||||
|
||||
const ( |
||||
QCUnknown QCResult = iota |
||||
QCYes |
||||
QCNo |
||||
QCMaybe |
||||
) |
||||
|
||||
func (r QCResult) String() string { |
||||
switch r { |
||||
case QCYes: |
||||
return "Yes" |
||||
case QCNo: |
||||
return "No" |
||||
case QCMaybe: |
||||
return "Maybe" |
||||
} |
||||
return "***UNKNOWN***" |
||||
} |
||||
|
||||
const ( |
||||
FCanonical = iota // NFC or NFD
|
||||
FCompatibility // NFKC or NFKD
|
||||
FNumberOfFormTypes |
||||
) |
||||
|
||||
const ( |
||||
MComposed = iota // NFC or NFKC
|
||||
MDecomposed // NFD or NFKD
|
||||
MNumberOfModes |
||||
) |
||||
|
||||
// This contains only the properties we're interested in.
|
||||
type Char struct { |
||||
name string |
||||
codePoint rune // if zero, this index is not a valid code point.
|
||||
ccc uint8 // canonical combining class
|
||||
origCCC uint8 |
||||
excludeInComp bool // from CompositionExclusions.txt
|
||||
compatDecomp bool // it has a compatibility expansion
|
||||
|
||||
nTrailingNonStarters uint8 |
||||
nLeadingNonStarters uint8 // must be equal to trailing if non-zero
|
||||
|
||||
forms [FNumberOfFormTypes]FormInfo // For FCanonical and FCompatibility
|
||||
|
||||
state State |
||||
} |
||||
|
||||
var chars = make([]Char, MaxChar+1) |
||||
var cccMap = make(map[uint8]uint8) |
||||
|
||||
func (c Char) String() string { |
||||
buf := new(bytes.Buffer) |
||||
|
||||
fmt.Fprintf(buf, "%U [%s]:\n", c.codePoint, c.name) |
||||
fmt.Fprintf(buf, " ccc: %v\n", c.ccc) |
||||
fmt.Fprintf(buf, " excludeInComp: %v\n", c.excludeInComp) |
||||
fmt.Fprintf(buf, " compatDecomp: %v\n", c.compatDecomp) |
||||
fmt.Fprintf(buf, " state: %v\n", c.state) |
||||
fmt.Fprintf(buf, " NFC:\n") |
||||
fmt.Fprint(buf, c.forms[FCanonical]) |
||||
fmt.Fprintf(buf, " NFKC:\n") |
||||
fmt.Fprint(buf, c.forms[FCompatibility]) |
||||
|
||||
return buf.String() |
||||
} |
||||
|
||||
// In UnicodeData.txt, some ranges are marked like this:
|
||||
// 3400;<CJK Ideograph Extension A, First>;Lo;0;L;;;;;N;;;;;
|
||||
// 4DB5;<CJK Ideograph Extension A, Last>;Lo;0;L;;;;;N;;;;;
|
||||
// parseCharacter keeps a state variable indicating the weirdness.
|
||||
type State int |
||||
|
||||
const ( |
||||
SNormal State = iota // known to be zero for the type
|
||||
SFirst |
||||
SLast |
||||
SMissing |
||||
) |
||||
|
||||
var lastChar = rune('\u0000') |
||||
|
||||
func (c Char) isValid() bool { |
||||
return c.codePoint != 0 && c.state != SMissing |
||||
} |
||||
|
||||
type FormInfo struct { |
||||
quickCheck [MNumberOfModes]QCResult // index: MComposed or MDecomposed
|
||||
verified [MNumberOfModes]bool // index: MComposed or MDecomposed
|
||||
|
||||
combinesForward bool // May combine with rune on the right
|
||||
combinesBackward bool // May combine with rune on the left
|
||||
isOneWay bool // Never appears in result
|
||||
inDecomp bool // Some decompositions result in this char.
|
||||
decomp Decomposition |
||||
expandedDecomp Decomposition |
||||
} |
||||
|
||||
func (f FormInfo) String() string { |
||||
buf := bytes.NewBuffer(make([]byte, 0)) |
||||
|
||||
fmt.Fprintf(buf, " quickCheck[C]: %v\n", f.quickCheck[MComposed]) |
||||
fmt.Fprintf(buf, " quickCheck[D]: %v\n", f.quickCheck[MDecomposed]) |
||||
fmt.Fprintf(buf, " cmbForward: %v\n", f.combinesForward) |
||||
fmt.Fprintf(buf, " cmbBackward: %v\n", f.combinesBackward) |
||||
fmt.Fprintf(buf, " isOneWay: %v\n", f.isOneWay) |
||||
fmt.Fprintf(buf, " inDecomp: %v\n", f.inDecomp) |
||||
fmt.Fprintf(buf, " decomposition: %X\n", f.decomp) |
||||
fmt.Fprintf(buf, " expandedDecomp: %X\n", f.expandedDecomp) |
||||
|
||||
return buf.String() |
||||
} |
||||
|
||||
type Decomposition []rune |
||||
|
||||
func parseDecomposition(s string, skipfirst bool) (a []rune, err error) { |
||||
decomp := strings.Split(s, " ") |
||||
if len(decomp) > 0 && skipfirst { |
||||
decomp = decomp[1:] |
||||
} |
||||
for _, d := range decomp { |
||||
point, err := strconv.ParseUint(d, 16, 64) |
||||
if err != nil { |
||||
return a, err |
||||
} |
||||
a = append(a, rune(point)) |
||||
} |
||||
return a, nil |
||||
} |
||||
|
||||
func loadUnicodeData() { |
||||
f := gen.OpenUCDFile("UnicodeData.txt") |
||||
defer f.Close() |
||||
p := ucd.New(f) |
||||
for p.Next() { |
||||
r := p.Rune(ucd.CodePoint) |
||||
char := &chars[r] |
||||
|
||||
char.ccc = uint8(p.Uint(ucd.CanonicalCombiningClass)) |
||||
decmap := p.String(ucd.DecompMapping) |
||||
|
||||
exp, err := parseDecomposition(decmap, false) |
||||
isCompat := false |
||||
if err != nil { |
||||
if len(decmap) > 0 { |
||||
exp, err = parseDecomposition(decmap, true) |
||||
if err != nil { |
||||
log.Fatalf(`%U: bad decomp |%v|: "%s"`, r, decmap, err) |
||||
} |
||||
isCompat = true |
||||
} |
||||
} |
||||
|
||||
char.name = p.String(ucd.Name) |
||||
char.codePoint = r |
||||
char.forms[FCompatibility].decomp = exp |
||||
if !isCompat { |
||||
char.forms[FCanonical].decomp = exp |
||||
} else { |
||||
char.compatDecomp = true |
||||
} |
||||
if len(decmap) > 0 { |
||||
char.forms[FCompatibility].decomp = exp |
||||
} |
||||
} |
||||
if err := p.Err(); err != nil { |
||||
log.Fatal(err) |
||||
} |
||||
} |
||||
|
||||
// compactCCC converts the sparse set of CCC values to a continguous one,
|
||||
// reducing the number of bits needed from 8 to 6.
|
||||
func compactCCC() { |
||||
m := make(map[uint8]uint8) |
||||
for i := range chars { |
||||
c := &chars[i] |
||||
m[c.ccc] = 0 |
||||
} |
||||
cccs := []int{} |
||||
for v, _ := range m { |
||||
cccs = append(cccs, int(v)) |
||||
} |
||||
sort.Ints(cccs) |
||||
for i, c := range cccs { |
||||
cccMap[uint8(i)] = uint8(c) |
||||
m[uint8(c)] = uint8(i) |
||||
} |
||||
for i := range chars { |
||||
c := &chars[i] |
||||
c.origCCC = c.ccc |
||||
c.ccc = m[c.ccc] |
||||
} |
||||
if len(m) >= 1<<6 { |
||||
log.Fatalf("too many difference CCC values: %d >= 64", len(m)) |
||||
} |
||||
} |
||||
|
||||
// CompositionExclusions.txt has form:
|
||||
// 0958 # ...
|
||||
// See http://unicode.org/reports/tr44/ for full explanation
|
||||
func loadCompositionExclusions() { |
||||
f := gen.OpenUCDFile("CompositionExclusions.txt") |
||||
defer f.Close() |
||||
p := ucd.New(f) |
||||
for p.Next() { |
||||
c := &chars[p.Rune(0)] |
||||
if c.excludeInComp { |
||||
log.Fatalf("%U: Duplicate entry in exclusions.", c.codePoint) |
||||
} |
||||
c.excludeInComp = true |
||||
} |
||||
if e := p.Err(); e != nil { |
||||
log.Fatal(e) |
||||
} |
||||
} |
||||
|
||||
// hasCompatDecomp returns true if any of the recursive
|
||||
// decompositions contains a compatibility expansion.
|
||||
// In this case, the character may not occur in NFK*.
|
||||
func hasCompatDecomp(r rune) bool { |
||||
c := &chars[r] |
||||
if c.compatDecomp { |
||||
return true |
||||
} |
||||
for _, d := range c.forms[FCompatibility].decomp { |
||||
if hasCompatDecomp(d) { |
||||
return true |
||||
} |
||||
} |
||||
return false |
||||
} |
||||
|
||||
// Hangul related constants.
|
||||
const ( |
||||
HangulBase = 0xAC00 |
||||
HangulEnd = 0xD7A4 // hangulBase + Jamo combinations (19 * 21 * 28)
|
||||
|
||||
JamoLBase = 0x1100 |
||||
JamoLEnd = 0x1113 |
||||
JamoVBase = 0x1161 |
||||
JamoVEnd = 0x1176 |
||||
JamoTBase = 0x11A8 |
||||
JamoTEnd = 0x11C3 |
||||
|
||||
JamoLVTCount = 19 * 21 * 28 |
||||
JamoTCount = 28 |
||||
) |
||||
|
||||
func isHangul(r rune) bool { |
||||
return HangulBase <= r && r < HangulEnd |
||||
} |
||||
|
||||
func isHangulWithoutJamoT(r rune) bool { |
||||
if !isHangul(r) { |
||||
return false |
||||
} |
||||
r -= HangulBase |
||||
return r < JamoLVTCount && r%JamoTCount == 0 |
||||
} |
||||
|
||||
func ccc(r rune) uint8 { |
||||
return chars[r].ccc |
||||
} |
||||
|
||||
// Insert a rune in a buffer, ordered by Canonical Combining Class.
|
||||
func insertOrdered(b Decomposition, r rune) Decomposition { |
||||
n := len(b) |
||||
b = append(b, 0) |
||||
cc := ccc(r) |
||||
if cc > 0 { |
||||
// Use bubble sort.
|
||||
for ; n > 0; n-- { |
||||
if ccc(b[n-1]) <= cc { |
||||
break |
||||
} |
||||
b[n] = b[n-1] |
||||
} |
||||
} |
||||
b[n] = r |
||||
return b |
||||
} |
||||
|
||||
// Recursively decompose.
|
||||
func decomposeRecursive(form int, r rune, d Decomposition) Decomposition { |
||||
dcomp := chars[r].forms[form].decomp |
||||
if len(dcomp) == 0 { |
||||
return insertOrdered(d, r) |
||||
} |
||||
for _, c := range dcomp { |
||||
d = decomposeRecursive(form, c, d) |
||||
} |
||||
return d |
||||
} |
||||
|
||||
func completeCharFields(form int) { |
||||
// Phase 0: pre-expand decomposition.
|
||||
for i := range chars { |
||||
f := &chars[i].forms[form] |
||||
if len(f.decomp) == 0 { |
||||
continue |
||||
} |
||||
exp := make(Decomposition, 0) |
||||
for _, c := range f.decomp { |
||||
exp = decomposeRecursive(form, c, exp) |
||||
} |
||||
f.expandedDecomp = exp |
||||
} |
||||
|
||||
// Phase 1: composition exclusion, mark decomposition.
|
||||
for i := range chars { |
||||
c := &chars[i] |
||||
f := &c.forms[form] |
||||
|
||||
// Marks script-specific exclusions and version restricted.
|
||||
f.isOneWay = c.excludeInComp |
||||
|
||||
// Singletons
|
||||
f.isOneWay = f.isOneWay || len(f.decomp) == 1 |
||||
|
||||
// Non-starter decompositions
|
||||
if len(f.decomp) > 1 { |
||||
chk := c.ccc != 0 || chars[f.decomp[0]].ccc != 0 |
||||
f.isOneWay = f.isOneWay || chk |
||||
} |
||||
|
||||
// Runes that decompose into more than two runes.
|
||||
f.isOneWay = f.isOneWay || len(f.decomp) > 2 |
||||
|
||||
if form == FCompatibility { |
||||
f.isOneWay = f.isOneWay || hasCompatDecomp(c.codePoint) |
||||
} |
||||
|
||||
for _, r := range f.decomp { |
||||
chars[r].forms[form].inDecomp = true |
||||
} |
||||
} |
||||
|
||||
// Phase 2: forward and backward combining.
|
||||
for i := range chars { |
||||
c := &chars[i] |
||||
f := &c.forms[form] |
||||
|
||||
if !f.isOneWay && len(f.decomp) == 2 { |
||||
f0 := &chars[f.decomp[0]].forms[form] |
||||
f1 := &chars[f.decomp[1]].forms[form] |
||||
if !f0.isOneWay { |
||||
f0.combinesForward = true |
||||
} |
||||
if !f1.isOneWay { |
||||
f1.combinesBackward = true |
||||
} |
||||
} |
||||
if isHangulWithoutJamoT(rune(i)) { |
||||
f.combinesForward = true |
||||
} |
||||
} |
||||
|
||||
// Phase 3: quick check values.
|
||||
for i := range chars { |
||||
c := &chars[i] |
||||
f := &c.forms[form] |
||||
|
||||
switch { |
||||
case len(f.decomp) > 0: |
||||
f.quickCheck[MDecomposed] = QCNo |
||||
case isHangul(rune(i)): |
||||
f.quickCheck[MDecomposed] = QCNo |
||||
default: |
||||
f.quickCheck[MDecomposed] = QCYes |
||||
} |
||||
switch { |
||||
case f.isOneWay: |
||||
f.quickCheck[MComposed] = QCNo |
||||
case (i & 0xffff00) == JamoLBase: |
||||
f.quickCheck[MComposed] = QCYes |
||||
if JamoLBase <= i && i < JamoLEnd { |
||||
f.combinesForward = true |
||||
} |
||||
if JamoVBase <= i && i < JamoVEnd { |
||||
f.quickCheck[MComposed] = QCMaybe |
||||
f.combinesBackward = true |
||||
f.combinesForward = true |
||||
} |
||||
if JamoTBase <= i && i < JamoTEnd { |
||||
f.quickCheck[MComposed] = QCMaybe |
||||
f.combinesBackward = true |
||||
} |
||||
case !f.combinesBackward: |
||||
f.quickCheck[MComposed] = QCYes |
||||
default: |
||||
f.quickCheck[MComposed] = QCMaybe |
||||
} |
||||
} |
||||
} |
||||
|
||||
func computeNonStarterCounts() { |
||||
// Phase 4: leading and trailing non-starter count
|
||||
for i := range chars { |
||||
c := &chars[i] |
||||
|
||||
runes := []rune{rune(i)} |
||||
// We always use FCompatibility so that the CGJ insertion points do not
|
||||
// change for repeated normalizations with different forms.
|
||||
if exp := c.forms[FCompatibility].expandedDecomp; len(exp) > 0 { |
||||
runes = exp |
||||
} |
||||
// We consider runes that combine backwards to be non-starters for the
|
||||
// purpose of Stream-Safe Text Processing.
|
||||
for _, r := range runes { |
||||
if cr := &chars[r]; cr.ccc == 0 && !cr.forms[FCompatibility].combinesBackward { |
||||
break |
||||
} |
||||
c.nLeadingNonStarters++ |
||||
} |
||||
for i := len(runes) - 1; i >= 0; i-- { |
||||
if cr := &chars[runes[i]]; cr.ccc == 0 && !cr.forms[FCompatibility].combinesBackward { |
||||
break |
||||
} |
||||
c.nTrailingNonStarters++ |
||||
} |
||||
if c.nTrailingNonStarters > 3 { |
||||
log.Fatalf("%U: Decomposition with more than 3 (%d) trailing modifiers (%U)", i, c.nTrailingNonStarters, runes) |
||||
} |
||||
|
||||
if isHangul(rune(i)) { |
||||
c.nTrailingNonStarters = 2 |
||||
if isHangulWithoutJamoT(rune(i)) { |
||||
c.nTrailingNonStarters = 1 |
||||
} |
||||
} |
||||
|
||||
if l, t := c.nLeadingNonStarters, c.nTrailingNonStarters; l > 0 && l != t { |
||||
log.Fatalf("%U: number of leading and trailing non-starters should be equal (%d vs %d)", i, l, t) |
||||
} |
||||
if t := c.nTrailingNonStarters; t > 3 { |
||||
log.Fatalf("%U: number of trailing non-starters is %d > 3", t) |
||||
} |
||||
} |
||||
} |
||||
|
||||
func printBytes(w io.Writer, b []byte, name string) { |
||||
fmt.Fprintf(w, "// %s: %d bytes\n", name, len(b)) |
||||
fmt.Fprintf(w, "var %s = [...]byte {", name) |
||||
for i, c := range b { |
||||
switch { |
||||
case i%64 == 0: |
||||
fmt.Fprintf(w, "\n// Bytes %x - %x\n", i, i+63) |
||||
case i%8 == 0: |
||||
fmt.Fprintf(w, "\n") |
||||
} |
||||
fmt.Fprintf(w, "0x%.2X, ", c) |
||||
} |
||||
fmt.Fprint(w, "\n}\n\n") |
||||
} |
||||
|
||||
// See forminfo.go for format.
|
||||
func makeEntry(f *FormInfo, c *Char) uint16 { |
||||
e := uint16(0) |
||||
if r := c.codePoint; HangulBase <= r && r < HangulEnd { |
||||
e |= 0x40 |
||||
} |
||||
if f.combinesForward { |
||||
e |= 0x20 |
||||
} |
||||
if f.quickCheck[MDecomposed] == QCNo { |
||||
e |= 0x4 |
||||
} |
||||
switch f.quickCheck[MComposed] { |
||||
case QCYes: |
||||
case QCNo: |
||||
e |= 0x10 |
||||
case QCMaybe: |
||||
e |= 0x18 |
||||
default: |
||||
log.Fatalf("Illegal quickcheck value %v.", f.quickCheck[MComposed]) |
||||
} |
||||
e |= uint16(c.nTrailingNonStarters) |
||||
return e |
||||
} |
||||
|
||||
// decompSet keeps track of unique decompositions, grouped by whether
|
||||
// the decomposition is followed by a trailing and/or leading CCC.
|
||||
type decompSet [7]map[string]bool |
||||
|
||||
const ( |
||||
normalDecomp = iota |
||||
firstMulti |
||||
firstCCC |
||||
endMulti |
||||
firstLeadingCCC |
||||
firstCCCZeroExcept |
||||
firstStarterWithNLead |
||||
lastDecomp |
||||
) |
||||
|
||||
var cname = []string{"firstMulti", "firstCCC", "endMulti", "firstLeadingCCC", "firstCCCZeroExcept", "firstStarterWithNLead", "lastDecomp"} |
||||
|
||||
func makeDecompSet() decompSet { |
||||
m := decompSet{} |
||||
for i := range m { |
||||
m[i] = make(map[string]bool) |
||||
} |
||||
return m |
||||
} |
||||
func (m *decompSet) insert(key int, s string) { |
||||
m[key][s] = true |
||||
} |
||||
|
||||
func printCharInfoTables(w io.Writer) int { |
||||
mkstr := func(r rune, f *FormInfo) (int, string) { |
||||
d := f.expandedDecomp |
||||
s := string([]rune(d)) |
||||
if max := 1 << 6; len(s) >= max { |
||||
const msg = "%U: too many bytes in decomposition: %d >= %d" |
||||
log.Fatalf(msg, r, len(s), max) |
||||
} |
||||
head := uint8(len(s)) |
||||
if f.quickCheck[MComposed] != QCYes { |
||||
head |= 0x40 |
||||
} |
||||
if f.combinesForward { |
||||
head |= 0x80 |
||||
} |
||||
s = string([]byte{head}) + s |
||||
|
||||
lccc := ccc(d[0]) |
||||
tccc := ccc(d[len(d)-1]) |
||||
cc := ccc(r) |
||||
if cc != 0 && lccc == 0 && tccc == 0 { |
||||
log.Fatalf("%U: trailing and leading ccc are 0 for non-zero ccc %d", r, cc) |
||||
} |
||||
if tccc < lccc && lccc != 0 { |
||||
const msg = "%U: lccc (%d) must be <= tcc (%d)" |
||||
log.Fatalf(msg, r, lccc, tccc) |
||||
} |
||||
index := normalDecomp |
||||
nTrail := chars[r].nTrailingNonStarters |
||||
nLead := chars[r].nLeadingNonStarters |
||||
if tccc > 0 || lccc > 0 || nTrail > 0 { |
||||
tccc <<= 2 |
||||
tccc |= nTrail |
||||
s += string([]byte{tccc}) |
||||
index = endMulti |
||||
for _, r := range d[1:] { |
||||
if ccc(r) == 0 { |
||||
index = firstCCC |
||||
} |
||||
} |
||||
if lccc > 0 || nLead > 0 { |
||||
s += string([]byte{lccc}) |
||||
if index == firstCCC { |
||||
log.Fatalf("%U: multi-segment decomposition not supported for decompositions with leading CCC != 0", r) |
||||
} |
||||
index = firstLeadingCCC |
||||
} |
||||
if cc != lccc { |
||||
if cc != 0 { |
||||
log.Fatalf("%U: for lccc != ccc, expected ccc to be 0; was %d", r, cc) |
||||
} |
||||
index = firstCCCZeroExcept |
||||
} |
||||
} else if len(d) > 1 { |
||||
index = firstMulti |
||||
} |
||||
return index, s |
||||
} |
||||
|
||||
decompSet := makeDecompSet() |
||||
const nLeadStr = "\x00\x01" // 0-byte length and tccc with nTrail.
|
||||
decompSet.insert(firstStarterWithNLead, nLeadStr) |
||||
|
||||
// Store the uniqued decompositions in a byte buffer,
|
||||
// preceded by their byte length.
|
||||
for _, c := range chars { |
||||
for _, f := range c.forms { |
||||
if len(f.expandedDecomp) == 0 { |
||||
continue |
||||
} |
||||
if f.combinesBackward { |
||||
log.Fatalf("%U: combinesBackward and decompose", c.codePoint) |
||||
} |
||||
index, s := mkstr(c.codePoint, &f) |
||||
decompSet.insert(index, s) |
||||
} |
||||
} |
||||
|
||||
decompositions := bytes.NewBuffer(make([]byte, 0, 10000)) |
||||
size := 0 |
||||
positionMap := make(map[string]uint16) |
||||
decompositions.WriteString("\000") |
||||
fmt.Fprintln(w, "const (") |
||||
for i, m := range decompSet { |
||||
sa := []string{} |
||||
for s := range m { |
||||
sa = append(sa, s) |
||||
} |
||||
sort.Strings(sa) |
||||
for _, s := range sa { |
||||
p := decompositions.Len() |
||||
decompositions.WriteString(s) |
||||
positionMap[s] = uint16(p) |
||||
} |
||||
if cname[i] != "" { |
||||
fmt.Fprintf(w, "%s = 0x%X\n", cname[i], decompositions.Len()) |
||||
} |
||||
} |
||||
fmt.Fprintln(w, "maxDecomp = 0x8000") |
||||
fmt.Fprintln(w, ")") |
||||
b := decompositions.Bytes() |
||||
printBytes(w, b, "decomps") |
||||
size += len(b) |
||||
|
||||
varnames := []string{"nfc", "nfkc"} |
||||
for i := 0; i < FNumberOfFormTypes; i++ { |
||||
trie := triegen.NewTrie(varnames[i]) |
||||
|
||||
for r, c := range chars { |
||||
f := c.forms[i] |
||||
d := f.expandedDecomp |
||||
if len(d) != 0 { |
||||
_, key := mkstr(c.codePoint, &f) |
||||
trie.Insert(rune(r), uint64(positionMap[key])) |
||||
if c.ccc != ccc(d[0]) { |
||||
// We assume the lead ccc of a decomposition !=0 in this case.
|
||||
if ccc(d[0]) == 0 { |
||||
log.Fatalf("Expected leading CCC to be non-zero; ccc is %d", c.ccc) |
||||
} |
||||
} |
||||
} else if c.nLeadingNonStarters > 0 && len(f.expandedDecomp) == 0 && c.ccc == 0 && !f.combinesBackward { |
||||
// Handle cases where it can't be detected that the nLead should be equal
|
||||
// to nTrail.
|
||||
trie.Insert(c.codePoint, uint64(positionMap[nLeadStr])) |
||||
} else if v := makeEntry(&f, &c)<<8 | uint16(c.ccc); v != 0 { |
||||
trie.Insert(c.codePoint, uint64(0x8000|v)) |
||||
} |
||||
} |
||||
sz, err := trie.Gen(w, triegen.Compact(&normCompacter{name: varnames[i]})) |
||||
if err != nil { |
||||
log.Fatal(err) |
||||
} |
||||
size += sz |
||||
} |
||||
return size |
||||
} |
||||
|
||||
func contains(sa []string, s string) bool { |
||||
for _, a := range sa { |
||||
if a == s { |
||||
return true |
||||
} |
||||
} |
||||
return false |
||||
} |
||||
|
||||
func makeTables() { |
||||
w := &bytes.Buffer{} |
||||
|
||||
size := 0 |
||||
if *tablelist == "" { |
||||
return |
||||
} |
||||
list := strings.Split(*tablelist, ",") |
||||
if *tablelist == "all" { |
||||
list = []string{"recomp", "info"} |
||||
} |
||||
|
||||
// Compute maximum decomposition size.
|
||||
max := 0 |
||||
for _, c := range chars { |
||||
if n := len(string(c.forms[FCompatibility].expandedDecomp)); n > max { |
||||
max = n |
||||
} |
||||
} |
||||
|
||||
fmt.Fprintln(w, "const (") |
||||
fmt.Fprintln(w, "\t// Version is the Unicode edition from which the tables are derived.") |
||||
fmt.Fprintf(w, "\tVersion = %q\n", gen.UnicodeVersion()) |
||||
fmt.Fprintln(w) |
||||
fmt.Fprintln(w, "\t// MaxTransformChunkSize indicates the maximum number of bytes that Transform") |
||||
fmt.Fprintln(w, "\t// may need to write atomically for any Form. Making a destination buffer at") |
||||
fmt.Fprintln(w, "\t// least this size ensures that Transform can always make progress and that") |
||||
fmt.Fprintln(w, "\t// the user does not need to grow the buffer on an ErrShortDst.") |
||||
fmt.Fprintf(w, "\tMaxTransformChunkSize = %d+maxNonStarters*4\n", len(string(0x034F))+max) |
||||
fmt.Fprintln(w, ")\n") |
||||
|
||||
// Print the CCC remap table.
|
||||
size += len(cccMap) |
||||
fmt.Fprintf(w, "var ccc = [%d]uint8{", len(cccMap)) |
||||
for i := 0; i < len(cccMap); i++ { |
||||
if i%8 == 0 { |
||||
fmt.Fprintln(w) |
||||
} |
||||
fmt.Fprintf(w, "%3d, ", cccMap[uint8(i)]) |
||||
} |
||||
fmt.Fprintln(w, "\n}\n") |
||||
|
||||
if contains(list, "info") { |
||||
size += printCharInfoTables(w) |
||||
} |
||||
|
||||
if contains(list, "recomp") { |
||||
// Note that we use 32 bit keys, instead of 64 bit.
|
||||
// This clips the bits of three entries, but we know
|
||||
// this won't cause a collision. The compiler will catch
|
||||
// any changes made to UnicodeData.txt that introduces
|
||||
// a collision.
|
||||
// Note that the recomposition map for NFC and NFKC
|
||||
// are identical.
|
||||
|
||||
// Recomposition map
|
||||
nrentries := 0 |
||||
for _, c := range chars { |
||||
f := c.forms[FCanonical] |
||||
if !f.isOneWay && len(f.decomp) > 0 { |
||||
nrentries++ |
||||
} |
||||
} |
||||
sz := nrentries * 8 |
||||
size += sz |
||||
fmt.Fprintf(w, "// recompMap: %d bytes (entries only)\n", sz) |
||||
fmt.Fprintln(w, "var recompMap = map[uint32]rune{") |
||||
for i, c := range chars { |
||||
f := c.forms[FCanonical] |
||||
d := f.decomp |
||||
if !f.isOneWay && len(d) > 0 { |
||||
key := uint32(uint16(d[0]))<<16 + uint32(uint16(d[1])) |
||||
fmt.Fprintf(w, "0x%.8X: 0x%.4X,\n", key, i) |
||||
} |
||||
} |
||||
fmt.Fprintf(w, "}\n\n") |
||||
} |
||||
|
||||
fmt.Fprintf(w, "// Total size of tables: %dKB (%d bytes)\n", (size+512)/1024, size) |
||||
gen.WriteGoFile("tables.go", "norm", w.Bytes()) |
||||
} |
||||
|
||||
func printChars() { |
||||
if *verbose { |
||||
for _, c := range chars { |
||||
if !c.isValid() || c.state == SMissing { |
||||
continue |
||||
} |
||||
fmt.Println(c) |
||||
} |
||||
} |
||||
} |
||||
|
||||
// verifyComputed does various consistency tests.
|
||||
func verifyComputed() { |
||||
for i, c := range chars { |
||||
for _, f := range c.forms { |
||||
isNo := (f.quickCheck[MDecomposed] == QCNo) |
||||
if (len(f.decomp) > 0) != isNo && !isHangul(rune(i)) { |
||||
log.Fatalf("%U: NF*D QC must be No if rune decomposes", i) |
||||
} |
||||
|
||||
isMaybe := f.quickCheck[MComposed] == QCMaybe |
||||
if f.combinesBackward != isMaybe { |
||||
log.Fatalf("%U: NF*C QC must be Maybe if combinesBackward", i) |
||||
} |
||||
if len(f.decomp) > 0 && f.combinesForward && isMaybe { |
||||
log.Fatalf("%U: NF*C QC must be Yes or No if combinesForward and decomposes", i) |
||||
} |
||||
|
||||
if len(f.expandedDecomp) != 0 { |
||||
continue |
||||
} |
||||
if a, b := c.nLeadingNonStarters > 0, (c.ccc > 0 || f.combinesBackward); a != b { |
||||
// We accept these runes to be treated differently (it only affects
|
||||
// segment breaking in iteration, most likely on improper use), but
|
||||
// reconsider if more characters are added.
|
||||
// U+FF9E HALFWIDTH KATAKANA VOICED SOUND MARK;Lm;0;L;<narrow> 3099;;;;N;;;;;
|
||||
// U+FF9F HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK;Lm;0;L;<narrow> 309A;;;;N;;;;;
|
||||
// U+3133 HANGUL LETTER KIYEOK-SIOS;Lo;0;L;<compat> 11AA;;;;N;HANGUL LETTER GIYEOG SIOS;;;;
|
||||
// U+318E HANGUL LETTER ARAEAE;Lo;0;L;<compat> 11A1;;;;N;HANGUL LETTER ALAE AE;;;;
|
||||
// U+FFA3 HALFWIDTH HANGUL LETTER KIYEOK-SIOS;Lo;0;L;<narrow> 3133;;;;N;HALFWIDTH HANGUL LETTER GIYEOG SIOS;;;;
|
||||
// U+FFDC HALFWIDTH HANGUL LETTER I;Lo;0;L;<narrow> 3163;;;;N;;;;;
|
||||
if i != 0xFF9E && i != 0xFF9F && !(0x3133 <= i && i <= 0x318E) && !(0xFFA3 <= i && i <= 0xFFDC) { |
||||
log.Fatalf("%U: nLead was %v; want %v", i, a, b) |
||||
} |
||||
} |
||||
} |
||||
nfc := c.forms[FCanonical] |
||||
nfkc := c.forms[FCompatibility] |
||||
if nfc.combinesBackward != nfkc.combinesBackward { |
||||
log.Fatalf("%U: Cannot combine combinesBackward\n", c.codePoint) |
||||
} |
||||
} |
||||
} |
||||
|
||||
// Use values in DerivedNormalizationProps.txt to compare against the
|
||||
// values we computed.
|
||||
// DerivedNormalizationProps.txt has form:
|
||||
// 00C0..00C5 ; NFD_QC; N # ...
|
||||
// 0374 ; NFD_QC; N # ...
|
||||
// See http://unicode.org/reports/tr44/ for full explanation
|
||||
func testDerived() { |
||||
f := gen.OpenUCDFile("DerivedNormalizationProps.txt") |
||||
defer f.Close() |
||||
p := ucd.New(f) |
||||
for p.Next() { |
||||
r := p.Rune(0) |
||||
c := &chars[r] |
||||
|
||||
var ftype, mode int |
||||
qt := p.String(1) |
||||
switch qt { |
||||
case "NFC_QC": |
||||
ftype, mode = FCanonical, MComposed |
||||
case "NFD_QC": |
||||
ftype, mode = FCanonical, MDecomposed |
||||
case "NFKC_QC": |
||||
ftype, mode = FCompatibility, MComposed |
||||
case "NFKD_QC": |
||||
ftype, mode = FCompatibility, MDecomposed |
||||
default: |
||||
continue |
||||
} |
||||
var qr QCResult |
||||
switch p.String(2) { |
||||
case "Y": |
||||
qr = QCYes |
||||
case "N": |
||||
qr = QCNo |
||||
case "M": |
||||
qr = QCMaybe |
||||
default: |
||||
log.Fatalf(`Unexpected quick check value "%s"`, p.String(2)) |
||||
} |
||||
if got := c.forms[ftype].quickCheck[mode]; got != qr { |
||||
log.Printf("%U: FAILED %s (was %v need %v)\n", r, qt, got, qr) |
||||
} |
||||
c.forms[ftype].verified[mode] = true |
||||
} |
||||
if err := p.Err(); err != nil { |
||||
log.Fatal(err) |
||||
} |
||||
// Any unspecified value must be QCYes. Verify this.
|
||||
for i, c := range chars { |
||||
for j, fd := range c.forms { |
||||
for k, qr := range fd.quickCheck { |
||||
if !fd.verified[k] && qr != QCYes { |
||||
m := "%U: FAIL F:%d M:%d (was %v need Yes) %s\n" |
||||
log.Printf(m, i, j, k, qr, c.name) |
||||
} |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
var testHeader = `const ( |
||||
Yes = iota |
||||
No |
||||
Maybe |
||||
) |
||||
|
||||
type formData struct { |
||||
qc uint8 |
||||
combinesForward bool |
||||
decomposition string |
||||
} |
||||
|
||||
type runeData struct { |
||||
r rune |
||||
ccc uint8 |
||||
nLead uint8 |
||||
nTrail uint8 |
||||
f [2]formData // 0: canonical; 1: compatibility
|
||||
} |
||||
|
||||
func f(qc uint8, cf bool, dec string) [2]formData { |
||||
return [2]formData{{qc, cf, dec}, {qc, cf, dec}} |
||||
} |
||||
|
||||
func g(qc, qck uint8, cf, cfk bool, d, dk string) [2]formData { |
||||
return [2]formData{{qc, cf, d}, {qck, cfk, dk}} |
||||
} |
||||
|
||||
var testData = []runeData{ |
||||
` |
||||
|
||||
func printTestdata() { |
||||
type lastInfo struct { |
||||
ccc uint8 |
||||
nLead uint8 |
||||
nTrail uint8 |
||||
f string |
||||
} |
||||
|
||||
last := lastInfo{} |
||||
w := &bytes.Buffer{} |
||||
fmt.Fprintf(w, testHeader) |
||||
for r, c := range chars { |
||||
f := c.forms[FCanonical] |
||||
qc, cf, d := f.quickCheck[MComposed], f.combinesForward, string(f.expandedDecomp) |
||||
f = c.forms[FCompatibility] |
||||
qck, cfk, dk := f.quickCheck[MComposed], f.combinesForward, string(f.expandedDecomp) |
||||
s := "" |
||||
if d == dk && qc == qck && cf == cfk { |
||||
s = fmt.Sprintf("f(%s, %v, %q)", qc, cf, d) |
||||
} else { |
||||
s = fmt.Sprintf("g(%s, %s, %v, %v, %q, %q)", qc, qck, cf, cfk, d, dk) |
||||
} |
||||
current := lastInfo{c.ccc, c.nLeadingNonStarters, c.nTrailingNonStarters, s} |
||||
if last != current { |
||||
fmt.Fprintf(w, "\t{0x%x, %d, %d, %d, %s},\n", r, c.origCCC, c.nLeadingNonStarters, c.nTrailingNonStarters, s) |
||||
last = current |
||||
} |
||||
} |
||||
fmt.Fprintln(w, "}") |
||||
gen.WriteGoFile("data_test.go", "norm", w.Bytes()) |
||||
} |
||||
@ -1,117 +0,0 @@ |
||||
// Copyright 2011 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// +build ignore
|
||||
|
||||
// Trie table generator.
|
||||
// Used by make*tables tools to generate a go file with trie data structures
|
||||
// for mapping UTF-8 to a 16-bit value. All but the last byte in a UTF-8 byte
|
||||
// sequence are used to lookup offsets in the index table to be used for the
|
||||
// next byte. The last byte is used to index into a table with 16-bit values.
|
||||
|
||||
package main |
||||
|
||||
import ( |
||||
"fmt" |
||||
"io" |
||||
) |
||||
|
||||
const maxSparseEntries = 16 |
||||
|
||||
type normCompacter struct { |
||||
sparseBlocks [][]uint64 |
||||
sparseOffset []uint16 |
||||
sparseCount int |
||||
name string |
||||
} |
||||
|
||||
func mostFrequentStride(a []uint64) int { |
||||
counts := make(map[int]int) |
||||
var v int |
||||
for _, x := range a { |
||||
if stride := int(x) - v; v != 0 && stride >= 0 { |
||||
counts[stride]++ |
||||
} |
||||
v = int(x) |
||||
} |
||||
var maxs, maxc int |
||||
for stride, cnt := range counts { |
||||
if cnt > maxc || (cnt == maxc && stride < maxs) { |
||||
maxs, maxc = stride, cnt |
||||
} |
||||
} |
||||
return maxs |
||||
} |
||||
|
||||
func countSparseEntries(a []uint64) int { |
||||
stride := mostFrequentStride(a) |
||||
var v, count int |
||||
for _, tv := range a { |
||||
if int(tv)-v != stride { |
||||
if tv != 0 { |
||||
count++ |
||||
} |
||||
} |
||||
v = int(tv) |
||||
} |
||||
return count |
||||
} |
||||
|
||||
func (c *normCompacter) Size(v []uint64) (sz int, ok bool) { |
||||
if n := countSparseEntries(v); n <= maxSparseEntries { |
||||
return (n+1)*4 + 2, true |
||||
} |
||||
return 0, false |
||||
} |
||||
|
||||
func (c *normCompacter) Store(v []uint64) uint32 { |
||||
h := uint32(len(c.sparseOffset)) |
||||
c.sparseBlocks = append(c.sparseBlocks, v) |
||||
c.sparseOffset = append(c.sparseOffset, uint16(c.sparseCount)) |
||||
c.sparseCount += countSparseEntries(v) + 1 |
||||
return h |
||||
} |
||||
|
||||
func (c *normCompacter) Handler() string { |
||||
return c.name + "Sparse.lookup" |
||||
} |
||||
|
||||
func (c *normCompacter) Print(w io.Writer) (retErr error) { |
||||
p := func(f string, x ...interface{}) { |
||||
if _, err := fmt.Fprintf(w, f, x...); retErr == nil && err != nil { |
||||
retErr = err |
||||
} |
||||
} |
||||
|
||||
ls := len(c.sparseBlocks) |
||||
p("// %sSparseOffset: %d entries, %d bytes\n", c.name, ls, ls*2) |
||||
p("var %sSparseOffset = %#v\n\n", c.name, c.sparseOffset) |
||||
|
||||
ns := c.sparseCount |
||||
p("// %sSparseValues: %d entries, %d bytes\n", c.name, ns, ns*4) |
||||
p("var %sSparseValues = [%d]valueRange {", c.name, ns) |
||||
for i, b := range c.sparseBlocks { |
||||
p("\n// Block %#x, offset %#x", i, c.sparseOffset[i]) |
||||
var v int |
||||
stride := mostFrequentStride(b) |
||||
n := countSparseEntries(b) |
||||
p("\n{value:%#04x,lo:%#02x},", stride, uint8(n)) |
||||
for i, nv := range b { |
||||
if int(nv)-v != stride { |
||||
if v != 0 { |
||||
p(",hi:%#02x},", 0x80+i-1) |
||||
} |
||||
if nv != 0 { |
||||
p("\n{value:%#04x,lo:%#02x", nv, 0x80+i) |
||||
} |
||||
} |
||||
v = int(nv) |
||||
} |
||||
if v != 0 { |
||||
p(",hi:%#02x},", 0x80+len(b)-1) |
||||
} |
||||
} |
||||
p("\n}\n\n") |
||||
return |
||||
} |
||||
Loading…
Reference in new issue