diff options
Diffstat (limited to 'vendor/github.com/paulrosania/go-charset/charset/iconv')
3 files changed, 440 insertions, 0 deletions
diff --git a/vendor/github.com/paulrosania/go-charset/charset/iconv/iconv.go b/vendor/github.com/paulrosania/go-charset/charset/iconv/iconv.go new file mode 100644 index 00000000..f7187f5f --- /dev/null +++ b/vendor/github.com/paulrosania/go-charset/charset/iconv/iconv.go @@ -0,0 +1,184 @@ +// The iconv package provides an interface to the GNU iconv character set +// conversion library (see http://www.gnu.org/software/libiconv/). +// It automatically registers all the character sets with the charset package, +// so it is usually used simply for the side effects of importing it. +// Example: +// import ( +// "go-charset.googlecode.com/hg/charset" +// _ "go-charset.googlecode.com/hg/charset/iconv" +// ) +package iconv + +//#cgo darwin LDFLAGS: -liconv +//#include <stdlib.h> +//#include <iconv.h> +//#include <errno.h> +//iconv_t iconv_open_error = (iconv_t)-1; +//size_t iconv_error = (size_t)-1; +import "C" +import ( + "errors" + "fmt" + "github.com/paulrosania/go-charset/charset" + "runtime" + "strings" + "syscall" + "unicode/utf8" + "unsafe" +) + +type iconvTranslator struct { + cd C.iconv_t + invalid rune + scratch []byte +} + +func canonicalChar(c rune) rune { + if c >= 'a' && c <= 'z' { + return c - 'a' + 'A' + } + return c +} + +func canonicalName(s string) string { + return strings.Map(canonicalChar, s) +} + +func init() { + charset.Register(iconvFactory{}) +} + +type iconvFactory struct { +} + +func (iconvFactory) TranslatorFrom(name string) (charset.Translator, error) { + return Translator("UTF-8", name, utf8.RuneError) +} + +func (iconvFactory) TranslatorTo(name string) (charset.Translator, error) { + // BUG This is wrong. The target character set may not be ASCII + // compatible. There's no easy solution to this other than + // removing the offending code point. + return Translator(name, "UTF-8", '?') +} + +// Translator returns a Translator that translates between +// the named character sets. When an invalid multibyte +// character is found, the bytes in invalid are substituted instead. +func Translator(toCharset, fromCharset string, invalid rune) (charset.Translator, error) { + cto, cfrom := C.CString(toCharset), C.CString(fromCharset) + cd, err := C.iconv_open(cto, cfrom) + + C.free(unsafe.Pointer(cfrom)) + C.free(unsafe.Pointer(cto)) + + if cd == C.iconv_open_error { + if err == syscall.EINVAL { + return nil, errors.New("iconv: conversion not supported") + } + return nil, err + } + t := &iconvTranslator{cd: cd, invalid: invalid} + runtime.SetFinalizer(t, func(*iconvTranslator) { + C.iconv_close(cd) + }) + return t, nil +} + +func (iconvFactory) Names() []string { + all := aliases() + names := make([]string, 0, len(all)) + for name, aliases := range all { + if aliases[0] == name { + names = append(names, name) + } + } + return names +} + +func (iconvFactory) Info(name string) *charset.Charset { + name = strings.ToLower(name) + all := aliases() + a, ok := all[name] + if !ok { + return nil + } + return &charset.Charset{ + Name: name, + Aliases: a, + } +} + +func (p *iconvTranslator) Translate(data []byte, eof bool) (rn int, rd []byte, rerr error) { + n := 0 + p.scratch = p.scratch[:0] + for len(data) > 0 { + p.scratch = ensureCap(p.scratch, len(p.scratch)+len(data)*utf8.UTFMax) + cData := (*C.char)(unsafe.Pointer(&data[:1][0])) + nData := C.size_t(len(data)) + + ns := len(p.scratch) + cScratch := (*C.char)(unsafe.Pointer(&p.scratch[ns : ns+1][0])) + nScratch := C.size_t(cap(p.scratch) - ns) + r, err := C.iconv(p.cd, &cData, &nData, &cScratch, &nScratch) + + p.scratch = p.scratch[0 : cap(p.scratch)-int(nScratch)] + n += len(data) - int(nData) + data = data[len(data)-int(nData):] + + if r != C.iconv_error || err == nil { + return n, p.scratch, nil + } + switch err := err.(syscall.Errno); err { + case C.EILSEQ: + // invalid multibyte sequence - skip one byte and continue + p.scratch = appendRune(p.scratch, p.invalid) + n++ + data = data[1:] + case C.EINVAL: + // incomplete multibyte sequence + return n, p.scratch, nil + case C.E2BIG: + // output buffer not large enough; try again with larger buffer. + p.scratch = ensureCap(p.scratch, cap(p.scratch)+utf8.UTFMax) + default: + panic(fmt.Sprintf("unexpected error code: %v", err)) + } + } + return n, p.scratch, nil +} + +// ensureCap returns s with a capacity of at least n bytes. +// If cap(s) < n, then it returns a new copy of s with the +// required capacity. +func ensureCap(s []byte, n int) []byte { + if n <= cap(s) { + return s + } + // logic adapted from appendslice1 in runtime + m := cap(s) + if m == 0 { + m = n + } else { + for { + if m < 1024 { + m += m + } else { + m += m / 4 + } + if m >= n { + break + } + } + } + t := make([]byte, len(s), m) + copy(t, s) + return t +} + +func appendRune(buf []byte, r rune) []byte { + n := len(buf) + buf = ensureCap(buf, n+utf8.UTFMax) + nu := utf8.EncodeRune(buf[n:n+utf8.UTFMax], r) + return buf[0 : n+nu] +} diff --git a/vendor/github.com/paulrosania/go-charset/charset/iconv/list_query.go b/vendor/github.com/paulrosania/go-charset/charset/iconv/list_query.go new file mode 100644 index 00000000..cda03270 --- /dev/null +++ b/vendor/github.com/paulrosania/go-charset/charset/iconv/list_query.go @@ -0,0 +1,80 @@ +// +build !linux +// This file is systemdependent because not all versions +// of iconv have the iconvlist function. + +package iconv + +//#cgo darwin LDFLAGS: -liconv +//#cgo freebsd LDFLAGS: -liconv +//#cgo windows LDFLAGS: -liconv +//#include <stdlib.h> +//#include <string.h> +//#include <iconv.h> +//#include <errno.h> +// +//typedef struct nameList nameList; +//struct nameList { +// int n; +// char **names; +// nameList *next; +//}; +// +//int +//addNames(unsigned int n, const char *const *names, void *data) { +// // we can't call back to Go because of the stack size issue, +// // so copy all the names. +// nameList *hd, *e; +// int i; +// +// hd = data; +// e = malloc(sizeof(nameList)); +// e->n = n; +// e->names = malloc(sizeof(char*) * n); +// for(i = 0; i < n; i++){ +// e->names[i] = strdup(names[i]); +// } +// e->next = hd->next; +// hd->next = e; +// return 0; +//} +// +//nameList * +//listNames(void) { +// nameList hd; +// hd.next = 0; +// iconvlist(addNames, &hd); +// return hd.next; +//} +import "C" + +import ( + "strings" + "sync" + "unsafe" +) + +var getAliasesOnce sync.Once +var allAliases = map[string][]string{} + +func aliases() map[string][]string { + getAliasesOnce.Do(getAliases) + return allAliases +} + +func getAliases() { + var next *C.nameList + for p := C.listNames(); p != nil; p = next { + next = p.next + aliases := make([]string, p.n) + pnames := (*[1e9]*C.char)(unsafe.Pointer(p.names)) + for i := range aliases { + aliases[i] = strings.ToLower(C.GoString(pnames[i])) + C.free(unsafe.Pointer(pnames[i])) + } + C.free(unsafe.Pointer(p.names)) + C.free(unsafe.Pointer(p)) + for _, alias := range aliases { + allAliases[alias] = aliases + } + } +} diff --git a/vendor/github.com/paulrosania/go-charset/charset/iconv/list_static.go b/vendor/github.com/paulrosania/go-charset/charset/iconv/list_static.go new file mode 100644 index 00000000..edf9e28a --- /dev/null +++ b/vendor/github.com/paulrosania/go-charset/charset/iconv/list_static.go @@ -0,0 +1,176 @@ +// +build linux + +// We just use a list of names obtained from iconv on a platform +// that allows iconvlist. We could invoke the iconv command, +// but that might fail too, and it gives no information about aliases. + +package iconv + +import ( + "sync" +) + +func aliases() map[string][]string { + initAliasesOnce.Do(initAliases) + return allAliases +} + +var initAliasesOnce sync.Once +var allAliases map[string][]string + +func initAliases() { + allAliases = make(map[string][]string) + for _, a := range aliasData { + for _, alias := range a { + allAliases[alias] = a + } + } +} + +var aliasData = [][]string{ + {"437", "cp437", "ibm437", "cspc8codepage437"}, + {"850", "cp850", "ibm850", "cspc850multilingual"}, + {"852", "cp852", "ibm852", "cspcp852"}, + {"855", "cp855", "ibm855", "csibm855"}, + {"857", "cp857", "ibm857", "csibm857"}, + {"860", "cp860", "ibm860", "csibm860"}, + {"861", "cp-is", "cp861", "ibm861", "csibm861"}, + {"862", "cp862", "ibm862", "cspc862latinhebrew"}, + {"863", "cp863", "ibm863", "csibm863"}, + {"865", "cp865", "ibm865", "csibm865"}, + {"866", "cp866", "ibm866", "csibm866"}, + {"869", "cp-gr", "cp869", "ibm869", "csibm869"}, + {"ansi-x3.4-1968", "ansi-x3.4-1986", "ascii", "cp367", "ibm367", "iso-ir-6", "iso646-us", "iso-646.irv:1991", "us", "us-ascii", "csascii"}, + {"arabic", "asmo-708", "ecma-114", "iso-8859-6", "iso-ir-127", "iso8859-6", "iso-8859-6", "iso-8859-6:1987", "csisolatinarabic"}, + {"armscii-8"}, + {"atari", "atarist"}, + {"big5-2003"}, + {"big-5", "big-five", "big5", "bigfive", "cn-big5", "csbig5"}, + {"big5-hkscs:1999"}, + {"big5-hkscs:2001"}, + {"big5-hkscs", "big5-hkscs:2004", "big5hkscs"}, + {"c99"}, + {"chinese", "gb-2312-80", "iso-ir-58", "csiso58gb231280"}, + {"cn", "gb-1988-80", "iso-ir-57", "iso646-cn", "csiso57gb1988"}, + {"cn-gb", "euc-cn", "euccn", "gb2312", "csgb2312"}, + {"cn-gb-isoir165", "iso-ir-165"}, + {"cp1046"}, + {"cp1124"}, + {"cp1125"}, + {"cp1129"}, + {"cp1131"}, + {"cp1133", "ibm-cp1133"}, + {"cp1161", "ibm-1161", "ibm1161", "csibm1161"}, + {"cp1162", "ibm-1162", "ibm1162", "csibm1162"}, + {"cp1163", "ibm-1163", "ibm1163", "csibm1163"}, + {"cp1250", "ms-ee", "windows-1250"}, + {"cp1251", "ms-cyrl", "windows-1251"}, + {"cp1252", "ms-ansi", "windows-1252"}, + {"cp1253", "ms-greek", "windows-1253"}, + {"cp1254", "ms-turk", "windows-1254"}, + {"cp1255", "ms-hebr", "windows-1255"}, + {"cp1256", "ms-arab", "windows-1256"}, + {"cp1257", "winbaltrim", "windows-1257"}, + {"cp1258", "windows-1258"}, + {"cp1361", "johab"}, + {"cp154", "cyrillic-asian", "pt154", "ptcp154", "csptcp154"}, + {"cp737"}, + {"cp775", "ibm775", "cspc775baltic"}, + {"cp819", "ibm819", "iso-8859-1", "iso-ir-100", "iso8859-1", "iso-8859-1", "iso-8859-1:1987", "l1", "latin1", "csisolatin1"}, + {"cp853"}, + {"cp856"}, + {"cp858"}, + {"cp864", "ibm864", "csibm864"}, + {"cp874", "windows-874"}, + {"cp922"}, + {"cp932"}, + {"cp936", "ms936", "windows-936"}, + {"cp943"}, + {"cp949", "uhc"}, + {"cp950"}, + {"cyrillic", "iso-8859-5", "iso-ir-144", "iso8859-5", "iso-8859-5", "iso-8859-5:1988", "csisolatincyrillic"}, + {"dec-hanyu"}, + {"dec-kanji"}, + {"ecma-118", "elot-928", "greek", "greek8", "iso-8859-7", "iso-ir-126", "iso8859-7", "iso-8859-7", "iso-8859-7:1987", "iso-8859-7:2003", "csisolatingreek"}, + {"euc-jis-2004", "euc-jisx0213"}, + {"euc-jp", "eucjp", "extended-unix-code-packed-format-for-japanese", "cseucpkdfmtjapanese"}, + {"euc-kr", "euckr", "cseuckr"}, + {"euc-tw", "euctw", "cseuctw"}, + {"gb18030"}, + {"gbk"}, + {"georgian-academy"}, + {"georgian-ps"}, + {"hebrew", "iso-8859-8", "iso-ir-138", "iso8859-8", "iso-8859-8", "iso-8859-8:1988", "csisolatinhebrew"}, + {"hp-roman8", "r8", "roman8", "cshproman8"}, + {"hz", "hz-gb-2312"}, + {"iso-10646-ucs-2", "ucs-2", "csunicode"}, + {"iso-10646-ucs-4", "ucs-4", "csucs4"}, + {"iso-2022-cn", "csiso2022cn"}, + {"iso-2022-cn-ext"}, + {"iso-2022-jp-1"}, + {"iso-2022-jp-2004", "iso-2022-jp-3"}, + {"iso-2022-jp-2", "csiso2022jp2"}, + {"iso-2022-jp", "csiso2022jp"}, + {"iso-2022-kr", "csiso2022kr"}, + {"iso-8859-10", "iso-ir-157", "iso8859-10", "iso-8859-10", "iso-8859-10:1992", "l6", "latin6", "csisolatin6"}, + {"iso-8859-11", "iso8859-11", "iso-8859-11"}, + {"iso-8859-13", "iso-ir-179", "iso8859-13", "iso-8859-13", "l7", "latin7"}, + {"iso-8859-14", "iso-celtic", "iso-ir-199", "iso8859-14", "iso-8859-14", "iso-8859-14:1998", "l8", "latin8"}, + {"iso-8859-15", "iso-ir-203", "iso8859-15", "iso-8859-15", "iso-8859-15:1998", "latin-9"}, + {"iso-8859-16", "iso-ir-226", "iso8859-16", "iso-8859-16", "iso-8859-16:2001", "l10", "latin10"}, + {"iso-8859-2", "iso-ir-101", "iso8859-2", "iso-8859-2", "iso-8859-2:1987", "l2", "latin2", "csisolatin2"}, + {"iso-8859-3", "iso-ir-109", "iso8859-3", "iso-8859-3", "iso-8859-3:1988", "l3", "latin3", "csisolatin3"}, + {"iso-8859-4", "iso-ir-110", "iso8859-4", "iso-8859-4", "iso-8859-4:1988", "l4", "latin4", "csisolatin4"}, + {"iso-8859-9", "iso-ir-148", "iso8859-9", "iso-8859-9", "iso-8859-9:1989", "l5", "latin5", "csisolatin5"}, + {"iso-ir-149", "korean", "ksc-5601", "ks-c-5601-1987", "ks-c-5601-1989", "csksc56011987"}, + {"iso-ir-14", "iso646-jp", "jis-c6220-1969-ro", "jp", "csiso14jisc6220ro"}, + {"iso-ir-159", "jis-x0212", "jis-x0212-1990", "jis-x0212.1990-0", "x0212", "csiso159jisx02121990"}, + {"iso-ir-166", "tis-620", "tis620", "tis620-0", "tis620.2529-1", "tis620.2533-0", "tis620.2533-1"}, + {"iso-ir-230", "tds565"}, + {"iso-ir-87", "jis0208", "jis-c6226-1983", "jis-x0208", "jis-x0208-1983", "jis-x0208-1990", "x0208", "csiso87jisx0208"}, + {"java"}, + {"jisx0201-1976", "jis-x0201", "x0201", "cshalfwidthkatakana"}, + {"koi8-r", "cskoi8r"}, + {"koi8-ru"}, + {"koi8-t"}, + {"koi8-u"}, + {"kz-1048", "rk1048", "strk1048-2002", "cskz1048"}, + {"macarabic"}, + {"maccentraleurope"}, + {"maccroatian"}, + {"maccyrillic"}, + {"macgreek"}, + {"machebrew"}, + {"maciceland"}, + {"mac", "macintosh", "macroman", "csmacintosh"}, + {"macromania"}, + {"macthai"}, + {"macturkish"}, + {"macukraine"}, + {"ms-kanji", "shift-jis", "shift-jis", "sjis", "csshiftjis"}, + {" MS-Windows", "Japanese", "(cp932)"}, + {"mulelao-1"}, + {"nextstep"}, + {"riscos-latin1"}, + {"shift-jis-2004", "shift-jisx0213"}, + {"tcvn", "tcvn-5712", "tcvn5712-1", "tcvn5712-1:1993"}, + {"ucs-2be", "unicode-1-1", "unicodebig", "csunicode11"}, + {"ucs-2-internal"}, + {"ucs-2le", "unicodelittle"}, + {"ucs-2-swapped"}, + {"ucs-4be"}, + {"ucs-4-internal"}, + {"ucs-4le"}, + {"ucs-4-swapped"}, + {"unicode-1-1-utf-7", "utf-7", "csunicode11utf7"}, + {"utf-16"}, + {"utf-16be"}, + {"utf-16le"}, + {"utf-32"}, + {"utf-32be"}, + {"utf-32le"}, + {"utf-8"}, + {"utf-8-mac", "utf8-mac"}, + {"viscii", "viscii1.1-1", "csviscii"}, + {"windows-31j", "cp932"}, +} |