diff options
Diffstat (limited to 'vendor/golang.org/x/text/encoding/htmlindex')
| -rw-r--r-- | vendor/golang.org/x/text/encoding/htmlindex/gen.go | 170 | ||||
| -rw-r--r-- | vendor/golang.org/x/text/encoding/htmlindex/htmlindex.go | 86 | ||||
| -rw-r--r-- | vendor/golang.org/x/text/encoding/htmlindex/map.go | 105 | ||||
| -rw-r--r-- | vendor/golang.org/x/text/encoding/htmlindex/tables.go | 352 |
4 files changed, 713 insertions, 0 deletions
diff --git a/vendor/golang.org/x/text/encoding/htmlindex/gen.go b/vendor/golang.org/x/text/encoding/htmlindex/gen.go new file mode 100644 index 00000000..80a52f0d --- /dev/null +++ b/vendor/golang.org/x/text/encoding/htmlindex/gen.go @@ -0,0 +1,170 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "log" + "strings" + + "golang.org/x/text/internal/gen" +) + +type group struct { + Encodings []struct { + Labels []string + Name string + } +} + +func main() { + gen.Init() + + r := gen.Open("https://encoding.spec.whatwg.org", "whatwg", "encodings.json") + var groups []group + if err := json.NewDecoder(r).Decode(&groups); err != nil { + log.Fatalf("Error reading encodings.json: %v", err) + } + + w := &bytes.Buffer{} + fmt.Fprintln(w, "type htmlEncoding byte") + fmt.Fprintln(w, "const (") + for i, g := range groups { + for _, e := range g.Encodings { + key := strings.ToLower(e.Name) + name := consts[key] + if name == "" { + log.Fatalf("No const defined for %s.", key) + } + if i == 0 { + fmt.Fprintf(w, "%s htmlEncoding = iota\n", name) + } else { + fmt.Fprintf(w, "%s\n", name) + } + } + } + fmt.Fprintln(w, "numEncodings") + fmt.Fprint(w, ")\n\n") + + fmt.Fprintln(w, "var canonical = [numEncodings]string{") + for _, g := range groups { + for _, e := range g.Encodings { + fmt.Fprintf(w, "%q,\n", strings.ToLower(e.Name)) + } + } + fmt.Fprint(w, "}\n\n") + + fmt.Fprintln(w, "var nameMap = map[string]htmlEncoding{") + for _, g := range groups { + for _, e := range g.Encodings { + for _, l := range e.Labels { + key := strings.ToLower(e.Name) + name := consts[key] + fmt.Fprintf(w, "%q: %s,\n", l, name) + } + } + } + fmt.Fprint(w, "}\n\n") + + var tags []string + fmt.Fprintln(w, "var localeMap = []htmlEncoding{") + for _, loc := range locales { + tags = append(tags, loc.tag) + fmt.Fprintf(w, "%s, // %s \n", consts[loc.name], loc.tag) + } + fmt.Fprint(w, "}\n\n") + + fmt.Fprintf(w, "const locales = %q\n", strings.Join(tags, " ")) + + gen.WriteGoFile("tables.go", "htmlindex", w.Bytes()) +} + +// consts maps canonical encoding name to internal constant. +var consts = map[string]string{ + "utf-8": "utf8", + "ibm866": "ibm866", + "iso-8859-2": "iso8859_2", + "iso-8859-3": "iso8859_3", + "iso-8859-4": "iso8859_4", + "iso-8859-5": "iso8859_5", + "iso-8859-6": "iso8859_6", + "iso-8859-7": "iso8859_7", + "iso-8859-8": "iso8859_8", + "iso-8859-8-i": "iso8859_8I", + "iso-8859-10": "iso8859_10", + "iso-8859-13": "iso8859_13", + "iso-8859-14": "iso8859_14", + "iso-8859-15": "iso8859_15", + "iso-8859-16": "iso8859_16", + "koi8-r": "koi8r", + "koi8-u": "koi8u", + "macintosh": "macintosh", + "windows-874": "windows874", + "windows-1250": "windows1250", + "windows-1251": "windows1251", + "windows-1252": "windows1252", + "windows-1253": "windows1253", + "windows-1254": "windows1254", + "windows-1255": "windows1255", + "windows-1256": "windows1256", + "windows-1257": "windows1257", + "windows-1258": "windows1258", + "x-mac-cyrillic": "macintoshCyrillic", + "gbk": "gbk", + "gb18030": "gb18030", + // "hz-gb-2312": "hzgb2312", // Was removed from WhatWG + "big5": "big5", + "euc-jp": "eucjp", + "iso-2022-jp": "iso2022jp", + "shift_jis": "shiftJIS", + "euc-kr": "euckr", + "replacement": "replacement", + "utf-16be": "utf16be", + "utf-16le": "utf16le", + "x-user-defined": "xUserDefined", +} + +// locales is taken from +// https://html.spec.whatwg.org/multipage/syntax.html#encoding-sniffing-algorithm. +var locales = []struct{ tag, name string }{ + {"und", "windows-1252"}, // The default value. + {"ar", "windows-1256"}, + {"ba", "windows-1251"}, + {"be", "windows-1251"}, + {"bg", "windows-1251"}, + {"cs", "windows-1250"}, + {"el", "iso-8859-7"}, + {"et", "windows-1257"}, + {"fa", "windows-1256"}, + {"he", "windows-1255"}, + {"hr", "windows-1250"}, + {"hu", "iso-8859-2"}, + {"ja", "shift_jis"}, + {"kk", "windows-1251"}, + {"ko", "euc-kr"}, + {"ku", "windows-1254"}, + {"ky", "windows-1251"}, + {"lt", "windows-1257"}, + {"lv", "windows-1257"}, + {"mk", "windows-1251"}, + {"pl", "iso-8859-2"}, + {"ru", "windows-1251"}, + {"sah", "windows-1251"}, + {"sk", "windows-1250"}, + {"sl", "iso-8859-2"}, + {"sr", "windows-1251"}, + {"tg", "windows-1251"}, + {"th", "windows-874"}, + {"tr", "windows-1254"}, + {"tt", "windows-1251"}, + {"uk", "windows-1251"}, + {"vi", "windows-1258"}, + {"zh-hans", "gb18030"}, + {"zh-hant", "big5"}, +} diff --git a/vendor/golang.org/x/text/encoding/htmlindex/htmlindex.go b/vendor/golang.org/x/text/encoding/htmlindex/htmlindex.go new file mode 100644 index 00000000..70f2ac4b --- /dev/null +++ b/vendor/golang.org/x/text/encoding/htmlindex/htmlindex.go @@ -0,0 +1,86 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:generate go run gen.go + +// Package htmlindex maps character set encoding names to Encodings as +// recommended by the W3C for use in HTML 5. See http://www.w3.org/TR/encoding. +package htmlindex + +// TODO: perhaps have a "bare" version of the index (used by this package) that +// is not pre-loaded with all encodings. Global variables in encodings prevent +// the linker from being able to purge unneeded tables. This means that +// referencing all encodings, as this package does for the default index, links +// in all encodings unconditionally. +// +// This issue can be solved by either solving the linking issue (see +// https://github.com/golang/go/issues/6330) or refactoring the encoding tables +// (e.g. moving the tables to internal packages that do not use global +// variables). + +// TODO: allow canonicalizing names + +import ( + "errors" + "strings" + "sync" + + "golang.org/x/text/encoding" + "golang.org/x/text/encoding/internal/identifier" + "golang.org/x/text/language" +) + +var ( + errInvalidName = errors.New("htmlindex: invalid encoding name") + errUnknown = errors.New("htmlindex: unknown Encoding") + errUnsupported = errors.New("htmlindex: this encoding is not supported") +) + +var ( + matcherOnce sync.Once + matcher language.Matcher +) + +// LanguageDefault returns the canonical name of the default encoding for a +// given language. +func LanguageDefault(tag language.Tag) string { + matcherOnce.Do(func() { + tags := []language.Tag{} + for _, t := range strings.Split(locales, " ") { + tags = append(tags, language.MustParse(t)) + } + matcher = language.NewMatcher(tags) + }) + _, i, _ := matcher.Match(tag) + return canonical[localeMap[i]] // Default is Windows-1252. +} + +// Get returns an Encoding for one of the names listed in +// http://www.w3.org/TR/encoding using the Default Index. Matching is case- +// insensitive. +func Get(name string) (encoding.Encoding, error) { + x, ok := nameMap[strings.ToLower(strings.TrimSpace(name))] + if !ok { + return nil, errInvalidName + } + return encodings[x], nil +} + +// Name reports the canonical name of the given Encoding. It will return +// an error if e is not associated with a supported encoding scheme. +func Name(e encoding.Encoding) (string, error) { + id, ok := e.(identifier.Interface) + if !ok { + return "", errUnknown + } + mib, _ := id.ID() + if mib == 0 { + return "", errUnknown + } + v, ok := mibMap[mib] + if !ok { + return "", errUnsupported + } + return canonical[v], nil +} diff --git a/vendor/golang.org/x/text/encoding/htmlindex/map.go b/vendor/golang.org/x/text/encoding/htmlindex/map.go new file mode 100644 index 00000000..c6143904 --- /dev/null +++ b/vendor/golang.org/x/text/encoding/htmlindex/map.go @@ -0,0 +1,105 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package htmlindex + +import ( + "golang.org/x/text/encoding" + "golang.org/x/text/encoding/charmap" + "golang.org/x/text/encoding/internal/identifier" + "golang.org/x/text/encoding/japanese" + "golang.org/x/text/encoding/korean" + "golang.org/x/text/encoding/simplifiedchinese" + "golang.org/x/text/encoding/traditionalchinese" + "golang.org/x/text/encoding/unicode" +) + +// mibMap maps a MIB identifier to an htmlEncoding index. +var mibMap = map[identifier.MIB]htmlEncoding{ + identifier.UTF8: utf8, + identifier.UTF16BE: utf16be, + identifier.UTF16LE: utf16le, + identifier.IBM866: ibm866, + identifier.ISOLatin2: iso8859_2, + identifier.ISOLatin3: iso8859_3, + identifier.ISOLatin4: iso8859_4, + identifier.ISOLatinCyrillic: iso8859_5, + identifier.ISOLatinArabic: iso8859_6, + identifier.ISOLatinGreek: iso8859_7, + identifier.ISOLatinHebrew: iso8859_8, + identifier.ISO88598I: iso8859_8I, + identifier.ISOLatin6: iso8859_10, + identifier.ISO885913: iso8859_13, + identifier.ISO885914: iso8859_14, + identifier.ISO885915: iso8859_15, + identifier.ISO885916: iso8859_16, + identifier.KOI8R: koi8r, + identifier.KOI8U: koi8u, + identifier.Macintosh: macintosh, + identifier.MacintoshCyrillic: macintoshCyrillic, + identifier.Windows874: windows874, + identifier.Windows1250: windows1250, + identifier.Windows1251: windows1251, + identifier.Windows1252: windows1252, + identifier.Windows1253: windows1253, + identifier.Windows1254: windows1254, + identifier.Windows1255: windows1255, + identifier.Windows1256: windows1256, + identifier.Windows1257: windows1257, + identifier.Windows1258: windows1258, + identifier.XUserDefined: xUserDefined, + identifier.GBK: gbk, + identifier.GB18030: gb18030, + identifier.Big5: big5, + identifier.EUCPkdFmtJapanese: eucjp, + identifier.ISO2022JP: iso2022jp, + identifier.ShiftJIS: shiftJIS, + identifier.EUCKR: euckr, + identifier.Replacement: replacement, +} + +// encodings maps the internal htmlEncoding to an Encoding. +// TODO: consider using a reusable index in encoding/internal. +var encodings = [numEncodings]encoding.Encoding{ + utf8: unicode.UTF8, + ibm866: charmap.CodePage866, + iso8859_2: charmap.ISO8859_2, + iso8859_3: charmap.ISO8859_3, + iso8859_4: charmap.ISO8859_4, + iso8859_5: charmap.ISO8859_5, + iso8859_6: charmap.ISO8859_6, + iso8859_7: charmap.ISO8859_7, + iso8859_8: charmap.ISO8859_8, + iso8859_8I: charmap.ISO8859_8I, + iso8859_10: charmap.ISO8859_10, + iso8859_13: charmap.ISO8859_13, + iso8859_14: charmap.ISO8859_14, + iso8859_15: charmap.ISO8859_15, + iso8859_16: charmap.ISO8859_16, + koi8r: charmap.KOI8R, + koi8u: charmap.KOI8U, + macintosh: charmap.Macintosh, + windows874: charmap.Windows874, + windows1250: charmap.Windows1250, + windows1251: charmap.Windows1251, + windows1252: charmap.Windows1252, + windows1253: charmap.Windows1253, + windows1254: charmap.Windows1254, + windows1255: charmap.Windows1255, + windows1256: charmap.Windows1256, + windows1257: charmap.Windows1257, + windows1258: charmap.Windows1258, + macintoshCyrillic: charmap.MacintoshCyrillic, + gbk: simplifiedchinese.GBK, + gb18030: simplifiedchinese.GB18030, + big5: traditionalchinese.Big5, + eucjp: japanese.EUCJP, + iso2022jp: japanese.ISO2022JP, + shiftJIS: japanese.ShiftJIS, + euckr: korean.EUCKR, + replacement: encoding.Replacement, + utf16be: unicode.UTF16(unicode.BigEndian, unicode.IgnoreBOM), + utf16le: unicode.UTF16(unicode.LittleEndian, unicode.IgnoreBOM), + xUserDefined: charmap.XUserDefined, +} diff --git a/vendor/golang.org/x/text/encoding/htmlindex/tables.go b/vendor/golang.org/x/text/encoding/htmlindex/tables.go new file mode 100644 index 00000000..78950d3c --- /dev/null +++ b/vendor/golang.org/x/text/encoding/htmlindex/tables.go @@ -0,0 +1,352 @@ +// This file was generated by go generate; DO NOT EDIT + +package htmlindex + +type htmlEncoding byte + +const ( + utf8 htmlEncoding = iota + ibm866 + iso8859_2 + iso8859_3 + iso8859_4 + iso8859_5 + iso8859_6 + iso8859_7 + iso8859_8 + iso8859_8I + iso8859_10 + iso8859_13 + iso8859_14 + iso8859_15 + iso8859_16 + koi8r + koi8u + macintosh + windows874 + windows1250 + windows1251 + windows1252 + windows1253 + windows1254 + windows1255 + windows1256 + windows1257 + windows1258 + macintoshCyrillic + gbk + gb18030 + big5 + eucjp + iso2022jp + shiftJIS + euckr + replacement + utf16be + utf16le + xUserDefined + numEncodings +) + +var canonical = [numEncodings]string{ + "utf-8", + "ibm866", + "iso-8859-2", + "iso-8859-3", + "iso-8859-4", + "iso-8859-5", + "iso-8859-6", + "iso-8859-7", + "iso-8859-8", + "iso-8859-8-i", + "iso-8859-10", + "iso-8859-13", + "iso-8859-14", + "iso-8859-15", + "iso-8859-16", + "koi8-r", + "koi8-u", + "macintosh", + "windows-874", + "windows-1250", + "windows-1251", + "windows-1252", + "windows-1253", + "windows-1254", + "windows-1255", + "windows-1256", + "windows-1257", + "windows-1258", + "x-mac-cyrillic", + "gbk", + "gb18030", + "big5", + "euc-jp", + "iso-2022-jp", + "shift_jis", + "euc-kr", + "replacement", + "utf-16be", + "utf-16le", + "x-user-defined", +} + +var nameMap = map[string]htmlEncoding{ + "unicode-1-1-utf-8": utf8, + "utf-8": utf8, + "utf8": utf8, + "866": ibm866, + "cp866": ibm866, + "csibm866": ibm866, + "ibm866": ibm866, + "csisolatin2": iso8859_2, + "iso-8859-2": iso8859_2, + "iso-ir-101": iso8859_2, + "iso8859-2": iso8859_2, + "iso88592": iso8859_2, + "iso_8859-2": iso8859_2, + "iso_8859-2:1987": iso8859_2, + "l2": iso8859_2, + "latin2": iso8859_2, + "csisolatin3": iso8859_3, + "iso-8859-3": iso8859_3, + "iso-ir-109": iso8859_3, + "iso8859-3": iso8859_3, + "iso88593": iso8859_3, + "iso_8859-3": iso8859_3, + "iso_8859-3:1988": iso8859_3, + "l3": iso8859_3, + "latin3": iso8859_3, + "csisolatin4": iso8859_4, + "iso-8859-4": iso8859_4, + "iso-ir-110": iso8859_4, + "iso8859-4": iso8859_4, + "iso88594": iso8859_4, + "iso_8859-4": iso8859_4, + "iso_8859-4:1988": iso8859_4, + "l4": iso8859_4, + "latin4": iso8859_4, + "csisolatincyrillic": iso8859_5, + "cyrillic": iso8859_5, + "iso-8859-5": iso8859_5, + "iso-ir-144": iso8859_5, + "iso8859-5": iso8859_5, + "iso88595": iso8859_5, + "iso_8859-5": iso8859_5, + "iso_8859-5:1988": iso8859_5, + "arabic": iso8859_6, + "asmo-708": iso8859_6, + "csiso88596e": iso8859_6, + "csiso88596i": iso8859_6, + "csisolatinarabic": iso8859_6, + "ecma-114": iso8859_6, + "iso-8859-6": iso8859_6, + "iso-8859-6-e": iso8859_6, + "iso-8859-6-i": iso8859_6, + "iso-ir-127": iso8859_6, + "iso8859-6": iso8859_6, + "iso88596": iso8859_6, + "iso_8859-6": iso8859_6, + "iso_8859-6:1987": iso8859_6, + "csisolatingreek": iso8859_7, + "ecma-118": iso8859_7, + "elot_928": iso8859_7, + "greek": iso8859_7, + "greek8": iso8859_7, + "iso-8859-7": iso8859_7, + "iso-ir-126": iso8859_7, + "iso8859-7": iso8859_7, + "iso88597": iso8859_7, + "iso_8859-7": iso8859_7, + "iso_8859-7:1987": iso8859_7, + "sun_eu_greek": iso8859_7, + "csiso88598e": iso8859_8, + "csisolatinhebrew": iso8859_8, + "hebrew": iso8859_8, + "iso-8859-8": iso8859_8, + "iso-8859-8-e": iso8859_8, + "iso-ir-138": iso8859_8, + "iso8859-8": iso8859_8, + "iso88598": iso8859_8, + "iso_8859-8": iso8859_8, + "iso_8859-8:1988": iso8859_8, + "visual": iso8859_8, + "csiso88598i": iso8859_8I, + "iso-8859-8-i": iso8859_8I, + "logical": iso8859_8I, + "csisolatin6": iso8859_10, + "iso-8859-10": iso8859_10, + "iso-ir-157": iso8859_10, + "iso8859-10": iso8859_10, + "iso885910": iso8859_10, + "l6": iso8859_10, + "latin6": iso8859_10, + "iso-8859-13": iso8859_13, + "iso8859-13": iso8859_13, + "iso885913": iso8859_13, + "iso-8859-14": iso8859_14, + "iso8859-14": iso8859_14, + "iso885914": iso8859_14, + "csisolatin9": iso8859_15, + "iso-8859-15": iso8859_15, + "iso8859-15": iso8859_15, + "iso885915": iso8859_15, + "iso_8859-15": iso8859_15, + "l9": iso8859_15, + "iso-8859-16": iso8859_16, + "cskoi8r": koi8r, + "koi": koi8r, + "koi8": koi8r, + "koi8-r": koi8r, + "koi8_r": koi8r, + "koi8-ru": koi8u, + "koi8-u": koi8u, + "csmacintosh": macintosh, + "mac": macintosh, + "macintosh": macintosh, + "x-mac-roman": macintosh, + "dos-874": windows874, + "iso-8859-11": windows874, + "iso8859-11": windows874, + "iso885911": windows874, + "tis-620": windows874, + "windows-874": windows874, + "cp1250": windows1250, + "windows-1250": windows1250, + "x-cp1250": windows1250, + "cp1251": windows1251, + "windows-1251": windows1251, + "x-cp1251": windows1251, + "ansi_x3.4-1968": windows1252, + "ascii": windows1252, + "cp1252": windows1252, + "cp819": windows1252, + "csisolatin1": windows1252, + "ibm819": windows1252, + "iso-8859-1": windows1252, + "iso-ir-100": windows1252, + "iso8859-1": windows1252, + "iso88591": windows1252, + "iso_8859-1": windows1252, + "iso_8859-1:1987": windows1252, + "l1": windows1252, + "latin1": windows1252, + "us-ascii": windows1252, + "windows-1252": windows1252, + "x-cp1252": windows1252, + "cp1253": windows1253, + "windows-1253": windows1253, + "x-cp1253": windows1253, + "cp1254": windows1254, + "csisolatin5": windows1254, + "iso-8859-9": windows1254, + "iso-ir-148": windows1254, + "iso8859-9": windows1254, + "iso88599": windows1254, + "iso_8859-9": windows1254, + "iso_8859-9:1989": windows1254, + "l5": windows1254, + "latin5": windows1254, + "windows-1254": windows1254, + "x-cp1254": windows1254, + "cp1255": windows1255, + "windows-1255": windows1255, + "x-cp1255": windows1255, + "cp1256": windows1256, + "windows-1256": windows1256, + "x-cp1256": windows1256, + "cp1257": windows1257, + "windows-1257": windows1257, + "x-cp1257": windows1257, + "cp1258": windows1258, + "windows-1258": windows1258, + "x-cp1258": windows1258, + "x-mac-cyrillic": macintoshCyrillic, + "x-mac-ukrainian": macintoshCyrillic, + "chinese": gbk, + "csgb2312": gbk, + "csiso58gb231280": gbk, + "gb2312": gbk, + "gb_2312": gbk, + "gb_2312-80": gbk, + "gbk": gbk, + "iso-ir-58": gbk, + "x-gbk": gbk, + "gb18030": gb18030, + "big5": big5, + "big5-hkscs": big5, + "cn-big5": big5, + "csbig5": big5, + "x-x-big5": big5, + "cseucpkdfmtjapanese": eucjp, + "euc-jp": eucjp, + "x-euc-jp": eucjp, + "csiso2022jp": iso2022jp, + "iso-2022-jp": iso2022jp, + "csshiftjis": shiftJIS, + "ms932": shiftJIS, + "ms_kanji": shiftJIS, + "shift-jis": shiftJIS, + "shift_jis": shiftJIS, + "sjis": shiftJIS, + "windows-31j": shiftJIS, + "x-sjis": shiftJIS, + "cseuckr": euckr, + "csksc56011987": euckr, + "euc-kr": euckr, + "iso-ir-149": euckr, + "korean": euckr, + "ks_c_5601-1987": euckr, + "ks_c_5601-1989": euckr, + "ksc5601": euckr, + "ksc_5601": euckr, + "windows-949": euckr, + "csiso2022kr": replacement, + "hz-gb-2312": replacement, + "iso-2022-cn": replacement, + "iso-2022-cn-ext": replacement, + "iso-2022-kr": replacement, + "utf-16be": utf16be, + "utf-16": utf16le, + "utf-16le": utf16le, + "x-user-defined": xUserDefined, +} + +var localeMap = []htmlEncoding{ + windows1252, // und + windows1256, // ar + windows1251, // ba + windows1251, // be + windows1251, // bg + windows1250, // cs + iso8859_7, // el + windows1257, // et + windows1256, // fa + windows1255, // he + windows1250, // hr + iso8859_2, // hu + shiftJIS, // ja + windows1251, // kk + euckr, // ko + windows1254, // ku + windows1251, // ky + windows1257, // lt + windows1257, // lv + windows1251, // mk + iso8859_2, // pl + windows1251, // ru + windows1251, // sah + windows1250, // sk + iso8859_2, // sl + windows1251, // sr + windows1251, // tg + windows874, // th + windows1254, // tr + windows1251, // tt + windows1251, // uk + windows1258, // vi + gb18030, // zh-hans + big5, // zh-hant +} + +const locales = "und ar ba be bg cs el et fa he hr hu ja kk ko ku ky lt lv mk pl ru sah sk sl sr tg th tr tt uk vi zh-hans zh-hant" |