Add vendor github.com/dfordsoft/golib/ic

1 files changed, 204 insertions, 583 deletions

diff --git a/vendor/golang.org/x/text/language/language.go b/vendor/golang.org/x/text/language/language.go
index 5eecceb6..f8923546 100644
--- a/vendor/golang.org/x/text/language/language.go
+++ b/vendor/golang.org/x/text/language/language.go
@@ -2,144 +2,42 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-//go:generate go run maketables.go gen_common.go -output tables.go
-//go:generate go run gen_index.go
+//go:generate go run gen.go -output tables.go
 
-// Package language implements BCP 47 language tags and related functionality.
-//
-// The Tag type, which is used to represent languages, is agnostic to the
-// meaning of its subtags. Tags are not fully canonicalized to preserve
-// information that may be valuable in certain contexts. As a consequence, two
-// different tags may represent identical languages.
-//
-// Initializing language- or locale-specific components usually consists of
-// two steps. The first step is to select a display language based on the
-// preferred languages of the user and the languages supported by an application.
-// The second step is to create the language-specific services based on
-// this selection. Each is discussed in more details below.
-//
-// Matching preferred against supported languages
-//
-// An application may support various languages. This list is typically limited
-// by the languages for which there exists translations of the user interface.
-// Similarly, a user may provide a list of preferred languages which is limited
-// by the languages understood by this user.
-// An application should use a Matcher to find the best supported language based
-// on the user's preferred list.
-// Matchers are aware of the intricacies of equivalence between languages.
-// The default Matcher implementation takes into account things such as
-// deprecated subtags, legacy tags, and mutual intelligibility between scripts
-// and languages.
-//
-// A Matcher for English, Australian English, Danish, and standard Mandarin can
-// be defined as follows:
-//
-//		var matcher = language.NewMatcher([]language.Tag{
-//			language.English,   // The first language is used as fallback.
-// 			language.MustParse("en-AU"),
-//			language.Danish,
-//			language.Chinese,
-//		})
-//
-// The following code selects the best match for someone speaking Spanish and
-// Norwegian:
-//
-// 		preferred := []language.Tag{ language.Spanish, language.Norwegian }
-//		tag, _, _ := matcher.Match(preferred...)
-//
-// In this case, the best match is Danish, as Danish is sufficiently a match to
-// Norwegian to not have to fall back to the default.
-// See ParseAcceptLanguage on how to handle the Accept-Language HTTP header.
-//
-// Selecting language-specific services
-//
-// One should always use the Tag returned by the Matcher to create an instance
-// of any of the language-specific services provided by the text repository.
-// This prevents the mixing of languages, such as having a different language for
-// messages and display names, as well as improper casing or sorting order for
-// the selected language.
-// Using the returned Tag also allows user-defined settings, such as collation
-// order or numbering system to be transparently passed as options.
-//
-// If you have language-specific data in your application, however, it will in
-// most cases suffice to use the index returned by the matcher to identify
-// the user language.
-// The following loop provides an alternative in case this is not sufficient:
-//
-// 		supported := map[language.Tag]data{
-//			language.English:            enData,
-// 			language.MustParse("en-AU"): enAUData,
-//			language.Danish:             daData,
-//			language.Chinese:            zhData,
-// 		}
-//		tag, _, _ := matcher.Match(preferred...)
-//		for ; tag != language.Und; tag = tag.Parent() {
-//			if v, ok := supported[tag]; ok {
-//				return v
-//			}
-//		}
-// 		return enData // should not reach here
-//
-// Repeatedly taking the Parent of the tag returned by Match will eventually
-// match one of the tags used to initialize the Matcher.
-//
-// Canonicalization
-//
-// By default, only legacy and deprecated tags are converted into their
-// canonical equivalent. All other information is preserved. This approach makes
-// the confidence scores more accurate and allows matchers to distinguish
-// between variants that are otherwise lost.
-//
-// As a consequence, two tags that should be treated as identical according to
-// BCP 47 or CLDR, like "en-Latn" and "en", will be represented differently. The
-// Matchers will handle such distinctions, though, and are aware of the
-// equivalence relations. The CanonType type can be used to alter the
-// canonicalization form.
-//
-// References
-//
-// BCP 47 - Tags for Identifying Languages
-// http://tools.ietf.org/html/bcp47
-package language // import "golang.org/x/text/language"
+package language
 
 // TODO: Remove above NOTE after:
 // - verifying that tables are dropped correctly (most notably matcher tables).
 
 import (
-	"errors"
-	"fmt"
 	"strings"
-)
 
-const (
-	// maxCoreSize is the maximum size of a BCP 47 tag without variants and
-	// extensions. Equals max lang (3) + script (4) + max reg (3) + 2 dashes.
-	maxCoreSize = 12
-
-	// max99thPercentileSize is a somewhat arbitrary buffer size that presumably
-	// is large enough to hold at least 99% of the BCP 47 tags.
-	max99thPercentileSize = 32
-
-	// maxSimpleUExtensionSize is the maximum size of a -u extension with one
-	// key-type pair. Equals len("-u-") + key (2) + dash + max value (8).
-	maxSimpleUExtensionSize = 14
+	"golang.org/x/text/internal/language"
+	"golang.org/x/text/internal/language/compact"
 )
 
 // Tag represents a BCP 47 language tag. It is used to specify an instance of a
 // specific language or locale. All language tag values are guaranteed to be
 // well-formed.
-type Tag struct {
-	lang     langID
-	region   regionID
-	script   scriptID
-	pVariant byte   // offset in str, includes preceding '-'
-	pExt     uint16 // offset of first extension, includes preceding '-'
+type Tag compact.Tag
+
+func makeTag(t language.Tag) (tag Tag) {
+	return Tag(compact.Make(t))
+}
 
-	// str is the string representation of the Tag. It will only be used if the
-	// tag has variants or extensions.
-	str string
+func (t *Tag) tag() language.Tag {
+	return (*compact.Tag)(t).Tag()
 }
 
+func (t *Tag) isCompact() bool {
+	return (*compact.Tag)(t).IsCompact()
+}
+
+// TODO: improve performance.
+func (t *Tag) lang() language.Language { return t.tag().LangID }
+func (t *Tag) region() language.Region { return t.tag().RegionID }
+func (t *Tag) script() language.Script { return t.tag().ScriptID }
+
 // Make is a convenience wrapper for Parse that omits the error.
 // In case of an error, a sensible default is returned.
 func Make(s string) Tag {
@@ -156,25 +54,13 @@ func (c CanonType) Make(s string) Tag {
 // Raw returns the raw base language, script and region, without making an
 // attempt to infer their values.
 func (t Tag) Raw() (b Base, s Script, r Region) {
-	return Base{t.lang}, Script{t.script}, Region{t.region}
-}
-
-// equalTags compares language, script and region subtags only.
-func (t Tag) equalTags(a Tag) bool {
-	return t.lang == a.lang && t.script == a.script && t.region == a.region
+	tt := t.tag()
+	return Base{tt.LangID}, Script{tt.ScriptID}, Region{tt.RegionID}
 }
 
 // IsRoot returns true if t is equal to language "und".
 func (t Tag) IsRoot() bool {
-	if int(t.pVariant) < len(t.str) {
-		return false
-	}
-	return t.equalTags(und)
-}
-
-// private reports whether the Tag consists solely of a private use tag.
-func (t Tag) private() bool {
-	return t.str != "" && t.pVariant == 0
+	return compact.Tag(t).IsRoot()
 }
 
 // CanonType can be used to enable or disable various types of canonicalization.
@@ -226,30 +112,30 @@ const (
 
 // canonicalize returns the canonicalized equivalent of the tag and
 // whether there was any change.
-func (t Tag) canonicalize(c CanonType) (Tag, bool) {
+func canonicalize(c CanonType, t language.Tag) (language.Tag, bool) {
 	if c == Raw {
 		return t, false
 	}
 	changed := false
 	if c&SuppressScript != 0 {
-		if t.lang < langNoIndexOffset && uint8(t.script) == suppressScript[t.lang] {
-			t.script = 0
+		if t.LangID.SuppressScript() == t.ScriptID {
+			t.ScriptID = 0
 			changed = true
 		}
 	}
 	if c&canonLang != 0 {
 		for {
-			if l, aliasType := normLang(t.lang); l != t.lang {
+			if l, aliasType := t.LangID.Canonicalize(); l != t.LangID {
 				switch aliasType {
-				case langLegacy:
+				case language.Legacy:
 					if c&Legacy != 0 {
-						if t.lang == _sh && t.script == 0 {
-							t.script = _Latn
+						if t.LangID == _sh && t.ScriptID == 0 {
+							t.ScriptID = _Latn
 						}
-						t.lang = l
+						t.LangID = l
 						changed = true
 					}
-				case langMacro:
+				case language.Macro:
 					if c&Macro != 0 {
 						// We deviate here from CLDR. The mapping "nb" -> "no"
 						// qualifies as a typical Macro language mapping.  However,
@@ -260,39 +146,39 @@ func (t Tag) canonicalize(c CanonType) (Tag, bool) {
 						// http://unicode.org/cldr/trac/ticket/1790 for some of the
 						// practical implications. TODO: this check could be removed
 						// if CLDR adopts this change.
-						if c&CLDR == 0 || t.lang != _nb {
+						if c&CLDR == 0 || t.LangID != _nb {
 							changed = true
-							t.lang = l
+							t.LangID = l
 						}
 					}
-				case langDeprecated:
+				case language.Deprecated:
 					if c&DeprecatedBase != 0 {
-						if t.lang == _mo && t.region == 0 {
-							t.region = _MD
+						if t.LangID == _mo && t.RegionID == 0 {
+							t.RegionID = _MD
 						}
-						t.lang = l
+						t.LangID = l
 						changed = true
 						// Other canonicalization types may still apply.
 						continue
 					}
 				}
-			} else if c&Legacy != 0 && t.lang == _no && c&CLDR != 0 {
-				t.lang = _nb
+			} else if c&Legacy != 0 && t.LangID == _no && c&CLDR != 0 {
+				t.LangID = _nb
 				changed = true
 			}
 			break
 		}
 	}
 	if c&DeprecatedScript != 0 {
-		if t.script == _Qaai {
+		if t.ScriptID == _Qaai {
 			changed = true
-			t.script = _Zinh
+			t.ScriptID = _Zinh
 		}
 	}
 	if c&DeprecatedRegion != 0 {
-		if r := normRegion(t.region); r != 0 {
+		if r := t.RegionID.Canonicalize(); r != t.RegionID {
 			changed = true
-			t.region = r
+			t.RegionID = r
 		}
 	}
 	return t, changed
@@ -300,11 +186,20 @@ func (t Tag) canonicalize(c CanonType) (Tag, bool) {
 
 // Canonicalize returns the canonicalized equivalent of the tag.
 func (c CanonType) Canonicalize(t Tag) (Tag, error) {
-	t, changed := t.canonicalize(c)
-	if changed {
-		t.remakeString()
+	// First try fast path.
+	if t.isCompact() {
+		if _, changed := canonicalize(c, compact.Tag(t).Tag()); !changed {
+			return t, nil
+		}
+	}
+	// It is unlikely that one will canonicalize a tag after matching. So do
+	// a slow but simple approach here.
+	if tag, changed := canonicalize(c, t.tag()); changed {
+		tag.RemakeString()
+		return makeTag(tag), nil
 	}
 	return t, nil
+
 }
 
 // Confidence indicates the level of certainty for a given return value.
@@ -327,79 +222,38 @@ func (c Confidence) String() string {
 	return confName[c]
 }
 
-// remakeString is used to update t.str in case lang, script or region changed.
-// It is assumed that pExt and pVariant still point to the start of the
-// respective parts.
-func (t *Tag) remakeString() {
-	if t.str == "" {
-		return
-	}
-	extra := t.str[t.pVariant:]
-	if t.pVariant > 0 {
-		extra = extra[1:]
-	}
-	if t.equalTags(und) && strings.HasPrefix(extra, "x-") {
-		t.str = extra
-		t.pVariant = 0
-		t.pExt = 0
-		return
-	}
-	var buf [max99thPercentileSize]byte // avoid extra memory allocation in most cases.
-	b := buf[:t.genCoreBytes(buf[:])]
-	if extra != "" {
-		diff := len(b) - int(t.pVariant)
-		b = append(b, '-')
-		b = append(b, extra...)
-		t.pVariant = uint8(int(t.pVariant) + diff)
-		t.pExt = uint16(int(t.pExt) + diff)
-	} else {
-		t.pVariant = uint8(len(b))
-		t.pExt = uint16(len(b))
-	}
-	t.str = string(b)
+// String returns the canonical string representation of the language tag.
+func (t Tag) String() string {
+	return t.tag().String()
 }
 
-// genCoreBytes writes a string for the base languages, script and region tags
-// to the given buffer and returns the number of bytes written. It will never
-// write more than maxCoreSize bytes.
-func (t *Tag) genCoreBytes(buf []byte) int {
-	n := t.lang.stringToBuf(buf[:])
-	if t.script != 0 {
-		n += copy(buf[n:], "-")
-		n += copy(buf[n:], t.script.String())
-	}
-	if t.region != 0 {
-		n += copy(buf[n:], "-")
-		n += copy(buf[n:], t.region.String())
-	}
-	return n
+// MarshalText implements encoding.TextMarshaler.
+func (t Tag) MarshalText() (text []byte, err error) {
+	return t.tag().MarshalText()
 }
 
-// String returns the canonical string representation of the language tag.
-func (t Tag) String() string {
-	if t.str != "" {
-		return t.str
-	}
-	if t.script == 0 && t.region == 0 {
-		return t.lang.String()
-	}
-	buf := [maxCoreSize]byte{}
-	return string(buf[:t.genCoreBytes(buf[:])])
+// UnmarshalText implements encoding.TextUnmarshaler.
+func (t *Tag) UnmarshalText(text []byte) error {
+	var tag language.Tag
+	err := tag.UnmarshalText(text)
+	*t = makeTag(tag)
+	return err
 }
 
 // Base returns the base language of the language tag. If the base language is
 // unspecified, an attempt will be made to infer it from the context.
 // It uses a variant of CLDR's Add Likely Subtags algorithm. This is subject to change.
 func (t Tag) Base() (Base, Confidence) {
-	if t.lang != 0 {
-		return Base{t.lang}, Exact
+	if b := t.lang(); b != 0 {
+		return Base{b}, Exact
 	}
+	tt := t.tag()
 	c := High
-	if t.script == 0 && !(Region{t.region}).IsCountry() {
+	if tt.ScriptID == 0 && !tt.RegionID.IsCountry() {
 		c = Low
 	}
-	if tag, err := addTags(t); err == nil && tag.lang != 0 {
-		return Base{tag.lang}, c
+	if tag, err := tt.Maximize(); err == nil && tag.LangID != 0 {
+		return Base{tag.LangID}, c
 	}
 	return Base{0}, No
 }
@@ -419,28 +273,27 @@ func (t Tag) Base() (Base, Confidence) {
 // in the past.  Also, the script that is commonly used may change over time.
 // It uses a variant of CLDR's Add Likely Subtags algorithm. This is subject to change.
 func (t Tag) Script() (Script, Confidence) {
-	if t.script != 0 {
-		return Script{t.script}, Exact
-	}
-	sc, c := scriptID(_Zzzz), No
-	if t.lang < langNoIndexOffset {
-		if scr := scriptID(suppressScript[t.lang]); scr != 0 {
-			// Note: it is not always the case that a language with a suppress
-			// script value is only written in one script (e.g. kk, ms, pa).
-			if t.region == 0 {
-				return Script{scriptID(scr)}, High
-			}
-			sc, c = scr, High
+	if scr := t.script(); scr != 0 {
+		return Script{scr}, Exact
+	}
+	tt := t.tag()
+	sc, c := language.Script(_Zzzz), No
+	if scr := tt.LangID.SuppressScript(); scr != 0 {
+		// Note: it is not always the case that a language with a suppress
+		// script value is only written in one script (e.g. kk, ms, pa).
+		if tt.RegionID == 0 {
+			return Script{scr}, High
 		}
+		sc, c = scr, High
 	}
-	if tag, err := addTags(t); err == nil {
-		if tag.script != sc {
-			sc, c = tag.script, Low
+	if tag, err := tt.Maximize(); err == nil {
+		if tag.ScriptID != sc {
+			sc, c = tag.ScriptID, Low
 		}
 	} else {
-		t, _ = (Deprecated | Macro).Canonicalize(t)
-		if tag, err := addTags(t); err == nil && tag.script != sc {
-			sc, c = tag.script, Low
+		tt, _ = canonicalize(Deprecated|Macro, tt)
+		if tag, err := tt.Maximize(); err == nil && tag.ScriptID != sc {
+			sc, c = tag.ScriptID, Low
 		}
 	}
 	return Script{sc}, c
@@ -450,28 +303,31 @@ func (t Tag) Script() (Script, Confidence) {
 // infer a most likely candidate from the context.
 // It uses a variant of CLDR's Add Likely Subtags algorithm. This is subject to change.
 func (t Tag) Region() (Region, Confidence) {
-	if t.region != 0 {
-		return Region{t.region}, Exact
+	if r := t.region(); r != 0 {
+		return Region{r}, Exact
 	}
-	if t, err := addTags(t); err == nil {
-		return Region{t.region}, Low // TODO: differentiate between high and low.
+	tt := t.tag()
+	if tt, err := tt.Maximize(); err == nil {
+		return Region{tt.RegionID}, Low // TODO: differentiate between high and low.
 	}
-	t, _ = (Deprecated | Macro).Canonicalize(t)
-	if tag, err := addTags(t); err == nil {
-		return Region{tag.region}, Low
+	tt, _ = canonicalize(Deprecated|Macro, tt)
+	if tag, err := tt.Maximize(); err == nil {
+		return Region{tag.RegionID}, Low
 	}
 	return Region{_ZZ}, No // TODO: return world instead of undetermined?
 }
 
-// Variant returns the variants specified explicitly for this language tag.
+// Variants returns the variants specified explicitly for this language tag.
 // or nil if no variant was specified.
 func (t Tag) Variants() []Variant {
+	if !compact.Tag(t).MayHaveVariants() {
+		return nil
+	}
 	v := []Variant{}
-	if int(t.pVariant) < int(t.pExt) {
-		for x, str := "", t.str[t.pVariant:t.pExt]; str != ""; {
-			x, str = nextToken(str)
-			v = append(v, Variant{x})
-		}
+	x, str := "", t.tag().Variants()
+	for str != "" {
+		x, str = nextToken(str)
+		v = append(v, Variant{x})
 	}
 	return v
 }
@@ -480,56 +336,7 @@ func (t Tag) Variants() []Variant {
 // specific language are substituted with fields from the parent language.
 // The parent for a language may change for newer versions of CLDR.
 func (t Tag) Parent() Tag {
-	if t.str != "" {
-		// Strip the variants and extensions.
-		t, _ = Raw.Compose(t.Raw())
-		if t.region == 0 && t.script != 0 && t.lang != 0 {
-			base, _ := addTags(Tag{lang: t.lang})
-			if base.script == t.script {
-				return Tag{lang: t.lang}
-			}
-		}
-		return t
-	}
-	if t.lang != 0 {
-		if t.region != 0 {
-			maxScript := t.script
-			if maxScript == 0 {
-				max, _ := addTags(t)
-				maxScript = max.script
-			}
-
-			for i := range parents {
-				if langID(parents[i].lang) == t.lang && scriptID(parents[i].maxScript) == maxScript {
-					for _, r := range parents[i].fromRegion {
-						if regionID(r) == t.region {
-							return Tag{
-								lang:   t.lang,
-								script: scriptID(parents[i].script),
-								region: regionID(parents[i].toRegion),
-							}
-						}
-					}
-				}
-			}
-
-			// Strip the script if it is the default one.
-			base, _ := addTags(Tag{lang: t.lang})
-			if base.script != maxScript {
-				return Tag{lang: t.lang, script: maxScript}
-			}
-			return Tag{lang: t.lang}
-		} else if t.script != 0 {
-			// The parent for an base-script pair with a non-default script is
-			// "und" instead of the base language.
-			base, _ := addTags(Tag{lang: t.lang})
-			if base.script != t.script {
-				return und
-			}
-			return Tag{lang: t.lang}
-		}
-	}
-	return und
+	return Tag(compact.Tag(t).Parent())
 }
 
 // returns token t and the rest of the string.
@@ -555,17 +362,8 @@ func (e Extension) String() string {
 
 // ParseExtension parses s as an extension and returns it on success.
 func ParseExtension(s string) (e Extension, err error) {
-	scan := makeScannerString(s)
-	var end int
-	if n := len(scan.token); n != 1 {
-		return Extension{}, errSyntax
-	}
-	scan.toLower(0, len(scan.b))
-	end = parseExtension(&scan)
-	if end != len(s) {
-		return Extension{}, errSyntax
-	}
-	return Extension{string(scan.b)}, nil
+	ext, err := language.ParseExtension(s)
+	return Extension{ext}, err
 }
 
 // Type returns the one-byte extension type of e. It returns 0 for the zero
@@ -586,22 +384,20 @@ func (e Extension) Tokens() []string {
 // false for ok if t does not have the requested extension. The returned
 // extension will be invalid in this case.
 func (t Tag) Extension(x byte) (ext Extension, ok bool) {
-	for i := int(t.pExt); i < len(t.str)-1; {
-		var ext string
-		i, ext = getExtension(t.str, i)
-		if ext[0] == x {
-			return Extension{ext}, true
-		}
+	if !compact.Tag(t).MayHaveExtensions() {
+		return Extension{}, false
 	}
-	return Extension{}, false
+	e, ok := t.tag().Extension(x)
+	return Extension{e}, ok
 }
 
 // Extensions returns all extensions of t.
 func (t Tag) Extensions() []Extension {
+	if !compact.Tag(t).MayHaveExtensions() {
+		return nil
+	}
 	e := []Extension{}
-	for i := int(t.pExt); i < len(t.str)-1; {
-		var ext string
-		i, ext = getExtension(t.str, i)
+	for _, ext := range t.tag().Extensions() {
 		e = append(e, Extension{ext})
 	}
 	return e
@@ -612,256 +408,102 @@ func (t Tag) Extensions() []Extension {
 // http://www.unicode.org/reports/tr35/#Unicode_Language_and_Locale_Identifiers.
 // TypeForKey will traverse the inheritance chain to get the correct value.
 func (t Tag) TypeForKey(key string) string {
-	if start, end, _ := t.findTypeForKey(key); end != start {
-		return t.str[start:end]
+	if !compact.Tag(t).MayHaveExtensions() {
+		if key != "rg" && key != "va" {
+			return ""
+		}
 	}
-	return ""
+	return t.tag().TypeForKey(key)
 }
 
-var (
-	errPrivateUse       = errors.New("cannot set a key on a private use tag")
-	errInvalidArguments = errors.New("invalid key or type")
-)
-
 // SetTypeForKey returns a new Tag with the key set to type, where key and type
 // are of the allowed values defined for the Unicode locale extension ('u') in
 // http://www.unicode.org/reports/tr35/#Unicode_Language_and_Locale_Identifiers.
 // An empty value removes an existing pair with the same key.
 func (t Tag) SetTypeForKey(key, value string) (Tag, error) {
-	if t.private() {
-		return t, errPrivateUse
-	}
-	if len(key) != 2 {
-		return t, errInvalidArguments
-	}
-
-	// Remove the setting if value is "".
-	if value == "" {
-		start, end, _ := t.findTypeForKey(key)
-		if start != end {
-			// Remove key tag and leading '-'.
-			start -= 4
-
-			// Remove a possible empty extension.
-			if (end == len(t.str) || t.str[end+2] == '-') && t.str[start-2] == '-' {
-				start -= 2
-			}
-			if start == int(t.pVariant) && end == len(t.str) {
-				t.str = ""
-				t.pVariant, t.pExt = 0, 0
-			} else {
-				t.str = fmt.Sprintf("%s%s", t.str[:start], t.str[end:])
-			}
-		}
-		return t, nil
-	}
-
-	if len(value) < 3 || len(value) > 8 {
-		return t, errInvalidArguments
-	}
-
-	var (
-		buf    [maxCoreSize + maxSimpleUExtensionSize]byte
-		uStart int // start of the -u extension.
-	)
-
-	// Generate the tag string if needed.
-	if t.str == "" {
-		uStart = t.genCoreBytes(buf[:])
-		buf[uStart] = '-'
-		uStart++
-	}
-
-	// Create new key-type pair and parse it to verify.
-	b := buf[uStart:]
-	copy(b, "u-")
-	copy(b[2:], key)
-	b[4] = '-'
-	b = b[:5+copy(b[5:], value)]
-	scan := makeScanner(b)
-	if parseExtensions(&scan); scan.err != nil {
-		return t, scan.err
-	}
-
-	// Assemble the replacement string.
-	if t.str == "" {
-		t.pVariant, t.pExt = byte(uStart-1), uint16(uStart-1)
-		t.str = string(buf[:uStart+len(b)])
-	} else {
-		s := t.str
-		start, end, hasExt := t.findTypeForKey(key)
-		if start == end {
-			if hasExt {
-				b = b[2:]
-			}
-			t.str = fmt.Sprintf("%s-%s%s", s[:start], b, s[end:])
-		} else {
-			t.str = fmt.Sprintf("%s%s%s", s[:start], value, s[end:])
-		}
-	}
-	return t, nil
+	tt, err := t.tag().SetTypeForKey(key, value)
+	return makeTag(tt), err
 }
 
-// findKeyAndType returns the start and end position for the type corresponding
-// to key or the point at which to insert the key-value pair if the type
-// wasn't found. The hasExt return value reports whether an -u extension was present.
-// Note: the extensions are typically very small and are likely to contain
-// only one key-type pair.
-func (t Tag) findTypeForKey(key string) (start, end int, hasExt bool) {
-	p := int(t.pExt)
-	if len(key) != 2 || p == len(t.str) || p == 0 {
-		return p, p, false
-	}
-	s := t.str
-
-	// Find the correct extension.
-	for p++; s[p] != 'u'; p++ {
-		if s[p] > 'u' {
-			p--
-			return p, p, false
-		}
-		if p = nextExtension(s, p); p == len(s) {
-			return len(s), len(s), false
-		}
-	}
-	// Proceed to the hyphen following the extension name.
-	p++
-
-	// curKey is the key currently being processed.
-	curKey := ""
-
-	// Iterate over keys until we get the end of a section.
-	for {
-		// p points to the hyphen preceding the current token.
-		if p3 := p + 3; s[p3] == '-' {
-			// Found a key.
-			// Check whether we just processed the key that was requested.
-			if curKey == key {
-				return start, p, true
-			}
-			// Set to the next key and continue scanning type tokens.
-			curKey = s[p+1 : p3]
-			if curKey > key {
-				return p, p, true
-			}
-			// Start of the type token sequence.
-			start = p + 4
-			// A type is at least 3 characters long.
-			p += 7 // 4 + 3
-		} else {
-			// Attribute or type, which is at least 3 characters long.
-			p += 4
-		}
-		// p points past the third character of a type or attribute.
-		max := p + 5 // maximum length of token plus hyphen.
-		if len(s) < max {
-			max = len(s)
-		}
-		for ; p < max && s[p] != '-'; p++ {
-		}
-		// Bail if we have exhausted all tokens or if the next token starts
-		// a new extension.
-		if p == len(s) || s[p+2] == '-' {
-			if curKey == key {
-				return start, p, true
-			}
-			return p, p, true
-		}
-	}
-}
+// NumCompactTags is the number of compact tags. The maximum tag is
+// NumCompactTags-1.
+const NumCompactTags = compact.NumCompactTags
 
 // CompactIndex returns an index, where 0 <= index < NumCompactTags, for tags
-// for which data exists in the text repository. The index will change over time
-// and should not be stored in persistent storage. Extensions, except for the
-// 'va' type of the 'u' extension, are ignored. It will return 0, false if no
-// compact tag exists, where 0 is the index for the root language (Und).
-func CompactIndex(t Tag) (index int, ok bool) {
-	// TODO: perhaps give more frequent tags a lower index.
-	// TODO: we could make the indexes stable. This will excluded some
-	//       possibilities for optimization, so don't do this quite yet.
-	b, s, r := t.Raw()
-	if len(t.str) > 0 {
-		if strings.HasPrefix(t.str, "x-") {
-			// We have no entries for user-defined tags.
-			return 0, false
-		}
-		if uint16(t.pVariant) != t.pExt {
-			// There are no tags with variants and an u-va type.
-			if t.TypeForKey("va") != "" {
-				return 0, false
-			}
-			t, _ = Raw.Compose(b, s, r, t.Variants())
-		} else if _, ok := t.Extension('u'); ok {
-			// Strip all but the 'va' entry.
-			variant := t.TypeForKey("va")
-			t, _ = Raw.Compose(b, s, r)
-			t, _ = t.SetTypeForKey("va", variant)
-		}
-		if len(t.str) > 0 {
-			// We have some variants.
-			for i, s := range specialTags {
-				if s == t {
-					return i + 1, true
-				}
-			}
-			return 0, false
-		}
-	}
-	// No variants specified: just compare core components.
-	// The key has the form lllssrrr, where l, s, and r are nibbles for
-	// respectively the langID, scriptID, and regionID.
-	key := uint32(b.langID) << (8 + 12)
-	key |= uint32(s.scriptID) << 12
-	key |= uint32(r.regionID)
-	x, ok := coreTags[key]
-	return int(x), ok
+// for which data exists in the text repository.The index will change over time
+// and should not be stored in persistent storage. If t does not match a compact
+// index, exact will be false and the compact index will be returned for the
+// first match after repeatedly taking the Parent of t.
+func CompactIndex(t Tag) (index int, exact bool) {
+	id, exact := compact.LanguageID(compact.Tag(t))
+	return int(id), exact
 }
 
+var root = language.Tag{}
+
 // Base is an ISO 639 language code, used for encoding the base language
 // of a language tag.
 type Base struct {
-	langID
+	langID language.Language
 }
 
 // ParseBase parses a 2- or 3-letter ISO 639 code.
 // It returns a ValueError if s is a well-formed but unknown language identifier
 // or another error if another error occurred.
 func ParseBase(s string) (Base, error) {
-	if n := len(s); n < 2 || 3 < n {
-		return Base{}, errSyntax
-	}
-	var buf [3]byte
-	l, err := getLangID(buf[:copy(buf[:], s)])
+	l, err := language.ParseBase(s)
 	return Base{l}, err
 }
 
+// String returns the BCP 47 representation of the base language.
+func (b Base) String() string {
+	return b.langID.String()
+}
+
+// ISO3 returns the ISO 639-3 language code.
+func (b Base) ISO3() string {
+	return b.langID.ISO3()
+}
+
+// IsPrivateUse reports whether this language code is reserved for private use.
+func (b Base) IsPrivateUse() bool {
+	return b.langID.IsPrivateUse()
+}
+
 // Script is a 4-letter ISO 15924 code for representing scripts.
 // It is idiomatically represented in title case.
 type Script struct {
-	scriptID
+	scriptID language.Script
 }
 
 // ParseScript parses a 4-letter ISO 15924 code.
 // It returns a ValueError if s is a well-formed but unknown script identifier
 // or another error if another error occurred.
 func ParseScript(s string) (Script, error) {
-	if len(s) != 4 {
-		return Script{}, errSyntax
-	}
-	var buf [4]byte
-	sc, err := getScriptID(script, buf[:copy(buf[:], s)])
+	sc, err := language.ParseScript(s)
 	return Script{sc}, err
 }
 
+// String returns the script code in title case.
+// It returns "Zzzz" for an unspecified script.
+func (s Script) String() string {
+	return s.scriptID.String()
+}
+
+// IsPrivateUse reports whether this script code is reserved for private use.
+func (s Script) IsPrivateUse() bool {
+	return s.scriptID.IsPrivateUse()
+}
+
 // Region is an ISO 3166-1 or UN M.49 code for representing countries and regions.
 type Region struct {
-	regionID
+	regionID language.Region
 }
 
 // EncodeM49 returns the Region for the given UN M.49 code.
 // It returns an error if r is not a valid code.
 func EncodeM49(r int) (Region, error) {
-	rid, err := getRegionM49(r)
+	rid, err := language.EncodeM49(r)
 	return Region{rid}, err
 }
 
@@ -869,62 +511,54 @@ func EncodeM49(r int) (Region, error) {
 // It returns a ValueError if s is a well-formed but unknown region identifier
 // or another error if another error occurred.
 func ParseRegion(s string) (Region, error) {
-	if n := len(s); n < 2 || 3 < n {
-		return Region{}, errSyntax
-	}
-	var buf [3]byte
-	r, err := getRegionID(buf[:copy(buf[:], s)])
+	r, err := language.ParseRegion(s)
 	return Region{r}, err
 }
 
+// String returns the BCP 47 representation for the region.
+// It returns "ZZ" for an unspecified region.
+func (r Region) String() string {
+	return r.regionID.String()
+}
+
+// ISO3 returns the 3-letter ISO code of r.
+// Note that not all regions have a 3-letter ISO code.
+// In such cases this method returns "ZZZ".
+func (r Region) ISO3() string {
+	return r.regionID.String()
+}
+
+// M49 returns the UN M.49 encoding of r, or 0 if this encoding
+// is not defined for r.
+func (r Region) M49() int {
+	return r.regionID.M49()
+}
+
+// IsPrivateUse reports whether r has the ISO 3166 User-assigned status. This
+// may include private-use tags that are assigned by CLDR and used in this
+// implementation. So IsPrivateUse and IsCountry can be simultaneously true.
+func (r Region) IsPrivateUse() bool {
+	return r.regionID.IsPrivateUse()
+}
+
 // IsCountry returns whether this region is a country or autonomous area. This
 // includes non-standard definitions from CLDR.
 func (r Region) IsCountry() bool {
-	if r.regionID == 0 || r.IsGroup() || r.IsPrivateUse() && r.regionID != _XK {
-		return false
-	}
-	return true
+	return r.regionID.IsCountry()
 }
 
 // IsGroup returns whether this region defines a collection of regions. This
 // includes non-standard definitions from CLDR.
 func (r Region) IsGroup() bool {
-	if r.regionID == 0 {
-		return false
-	}
-	return int(regionInclusion[r.regionID]) < len(regionContainment)
+	return r.regionID.IsGroup()
 }
 
 // Contains returns whether Region c is contained by Region r. It returns true
 // if c == r.
 func (r Region) Contains(c Region) bool {
-	return r.regionID.contains(c.regionID)
+	return r.regionID.Contains(c.regionID)
 }
 
-func (r regionID) contains(c regionID) bool {
-	if r == c {
-		return true
-	}
-	g := regionInclusion[r]
-	if g >= nRegionGroups {
-		return false
-	}
-	m := regionContainment[g]
-
-	d := regionInclusion[c]
-	b := regionInclusionBits[d]
-
-	// A contained country may belong to multiple disjoint groups. Matching any
-	// of these indicates containment. If the contained region is a group, it
-	// must strictly be a subset.
-	if d >= nRegionGroups {
-		return b&m != 0
-	}
-	return b&^m == 0
-}
-
-var errNoTLD = errors.New("language: region is not a valid ccTLD")
-
 // TLD returns the country code top-level domain (ccTLD). UK is returned for GB.
 // In all other cases it returns either the region itself or an error.
 //
@@ -933,25 +567,15 @@ var errNoTLD = errors.New("language: region is not a valid ccTLD")
 // region will already be canonicalized it was obtained from a Tag that was
 // obtained using any of the default methods.
 func (r Region) TLD() (Region, error) {
-	// See http://en.wikipedia.org/wiki/Country_code_top-level_domain for the
-	// difference between ISO 3166-1 and IANA ccTLD.
-	if r.regionID == _GB {
-		r = Region{_UK}
-	}
-	if (r.typ() & ccTLD) == 0 {
-		return Region{}, errNoTLD
-	}
-	return r, nil
+	tld, err := r.regionID.TLD()
+	return Region{tld}, err
 }
 
 // Canonicalize returns the region or a possible replacement if the region is
 // deprecated. It will not return a replacement for deprecated regions that
 // are split into multiple regions.
 func (r Region) Canonicalize() Region {
-	if cr := normRegion(r.regionID); cr != 0 {
-		return Region{cr}
-	}
-	return r
+	return Region{r.regionID.Canonicalize()}
 }
 
 // Variant represents a registered variant of a language as defined by BCP 47.
@@ -962,11 +586,8 @@ type Variant struct {
 // ParseVariant parses and returns a Variant. An error is returned if s is not
 // a valid variant.
 func ParseVariant(s string) (Variant, error) {
-	s = strings.ToLower(s)
-	if _, ok := variantIndex[s]; ok {
-		return Variant{s}, nil
-	}
-	return Variant{}, mkErrInvalid([]byte(s))
+	v, err := language.ParseVariant(s)
+	return Variant{v.String()}, err
 }
 
 // String returns the string representation of the variant.