Use Go1.11 module (#5743)

* Migrate to go modules

* make vendor

* Update mvdan.cc/xurls

* make vendor

* Update code.gitea.io/git

* make fmt-check

* Update github.com/go-sql-driver/mysql

* make vendor

vendor/golang.org/x/text/language/Makefile

@@ -0,0 +1,16 @@
+# Copyright 2013 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+CLEANFILES+=maketables
+
+maketables: maketables.go
+	go build $^
+
+tables:	maketables
+	./maketables > tables.go
+	gofmt -w -s tables.go
+
+# Build (but do not run) maketables during testing,
+# just to make sure it still compiles.
+testshort: maketables

vendor/golang.org/x/text/language/doc.go

@@ -0,0 +1,102 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package language implements BCP 47 language tags and related functionality.
+//
+// The most important function of package language is to match a list of
+// user-preferred languages to a list of supported languages.
+// It alleviates the developer of dealing with the complexity of this process
+// and provides the user with the best experience
+// (see https://blog.golang.org/matchlang).
+//
+//
+// Matching preferred against supported languages
+//
+// A Matcher for an application that supports English, Australian English,
+// Danish, and standard Mandarin can be created as follows:
+//
+//    var matcher = language.NewMatcher([]language.Tag{
+//        language.English,   // The first language is used as fallback.
+//        language.MustParse("en-AU"),
+//        language.Danish,
+//        language.Chinese,
+//    })
+//
+// This list of supported languages is typically implied by the languages for
+// which there exists translations of the user interface.
+//
+// User-preferred languages usually come as a comma-separated list of BCP 47
+// language tags.
+// The MatchString finds best matches for such strings:
+//
+//    handler(w http.ResponseWriter, r *http.Request) {
+//        lang, _ := r.Cookie("lang")
+//        accept := r.Header.Get("Accept-Language")
+//        tag, _ := language.MatchStrings(matcher, lang.String(), accept)
+//
+//        // tag should now be used for the initialization of any
+//        // locale-specific service.
+//    }
+//
+// The Matcher's Match method can be used to match Tags directly.
+//
+// Matchers are aware of the intricacies of equivalence between languages, such
+// as deprecated subtags, legacy tags, macro languages, mutual
+// intelligibility between scripts and languages, and transparently passing
+// BCP 47 user configuration.
+// For instance, it will know that a reader of Bokmål Danish can read Norwegian
+// and will know that Cantonese ("yue") is a good match for "zh-HK".
+//
+//
+// Using match results
+//
+// To guarantee a consistent user experience to the user it is important to
+// use the same language tag for the selection of any locale-specific services.
+// For example, it is utterly confusing to substitute spelled-out numbers
+// or dates in one language in text of another language.
+// More subtly confusing is using the wrong sorting order or casing
+// algorithm for a certain language.
+//
+//    All the packages in x/text that provide locale-specific services
+//    (e.g. collate, cases) should be initialized with the tag that was
+//    obtained at the start of an interaction with the user.
+//
+// Note that Tag that is returned by Match and MatchString may differ from any
+// of the supported languages, as it may contain carried over settings from
+// the user tags.
+// This may be inconvenient when your application has some additional
+// locale-specific data for your supported languages.
+// Match and MatchString both return the index of the matched supported tag
+// to simplify associating such data with the matched tag.
+//
+//
+// Canonicalization
+//
+// If one uses the Matcher to compare languages one does not need to
+// worry about canonicalization.
+//
+// The meaning of a Tag varies per application. The language package
+// therefore delays canonicalization and preserves information as much
+// as possible. The Matcher, however, will always take into account that
+// two different tags may represent the same language.
+//
+// By default, only legacy and deprecated tags are converted into their
+// canonical equivalent. All other information is preserved. This approach makes
+// the confidence scores more accurate and allows matchers to distinguish
+// between variants that are otherwise lost.
+//
+// As a consequence, two tags that should be treated as identical according to
+// BCP 47 or CLDR, like "en-Latn" and "en", will be represented differently. The
+// Matcher handles such distinctions, though, and is aware of the
+// equivalence relations. The CanonType type can be used to alter the
+// canonicalization form.
+//
+// References
+//
+// BCP 47 - Tags for Identifying Languages http://tools.ietf.org/html/bcp47
+//
+package language // import "golang.org/x/text/language"
+
+// TODO: explanation on how to match languages for your own locale-specific
+// service.

vendor/golang.org/x/text/language/gen.go

@@ -698,8 +698,8 @@ func (b *builder) computeRegionGroups() {
 			b.groups[group] = index(len(b.groups))
 		}
 	}
-	if len(b.groups) > 32 {
-		log.Fatalf("only 32 groups supported, found %d", len(b.groups))
+	if len(b.groups) > 64 {
+		log.Fatalf("only 64 groups supported, found %d", len(b.groups))
 	}
 	b.writeConst("nRegionGroups", len(b.groups))
 }
@@ -1417,20 +1417,27 @@ func (b *builder) writeMatchData() {
 	}
 	b.writeSlice("regionToGroups", regionToGroups)
 
+	// maps language id to in- and out-of-group region.
+	paradigmLocales := [][3]uint16{}
+	locales := strings.Split(lm[0].ParadigmLocales[0].Locales, " ")
+	for i := 0; i < len(locales); i += 2 {
+		x := [3]uint16{}
+		for j := 0; j < 2; j++ {
+			pc := strings.SplitN(locales[i+j], "-", 2)
+			x[0] = b.langIndex(pc[0])
+			if len(pc) == 2 {
+				x[1+j] = uint16(b.region.index(pc[1]))
+			}
+		}
+		paradigmLocales = append(paradigmLocales, x)
+	}
+	b.writeSlice("paradigmLocales", paradigmLocales)
+
 	b.writeType(mutualIntelligibility{})
 	b.writeType(scriptIntelligibility{})
 	b.writeType(regionIntelligibility{})
 
-	matchLang := []mutualIntelligibility{{
-		// TODO: remove once CLDR is fixed.
-		want:     uint16(b.langIndex("sr")),
-		have:     uint16(b.langIndex("hr")),
-		distance: uint8(5),
-	}, {
-		want:     uint16(b.langIndex("sr")),
-		have:     uint16(b.langIndex("bs")),
-		distance: uint8(5),
-	}}
+	matchLang := []mutualIntelligibility{}
 	matchScript := []scriptIntelligibility{}
 	matchRegion := []regionIntelligibility{}
 	// Convert the languageMatch entries in lists keyed by desired language.
@@ -1490,8 +1497,14 @@ func (b *builder) writeMatchData() {
 			if desired == supported && desired == "*_*_*" {
 				continue
 			}
-			if desired != supported { // (Weird but correct.)
-				log.Fatalf("not supported: desired=%q; supported=%q", desired, supported)
+			if desired != supported {
+				// This is now supported by CLDR, but only one case, which
+				// should already be covered by paradigm locales. For instance,
+				// test case "und, en, en-GU, en-IN, en-GB ; en-ZA ; en-GB" in
+				// testdata/CLDRLocaleMatcherTest.txt tests this.
+				if supported != "en_*_GB" {
+					log.Fatalf("not supported: desired=%q; supported=%q", desired, supported)
+				}
 				continue
 			}
 			ri := regionIntelligibility{
@@ -1559,7 +1572,7 @@ func (b *builder) writeRegionInclusionData() {
 		}
 	}
 
-	regionContainment := make([]uint32, len(b.groups))
+	regionContainment := make([]uint64, len(b.groups))
 	for _, g := range b.groups {
 		l := containment[g]
 
@@ -1577,10 +1590,10 @@ func (b *builder) writeRegionInclusionData() {
 	b.writeSlice("regionContainment", regionContainment)
 
 	regionInclusion := make([]uint8, len(b.region.s))
-	bvs := make(map[uint32]index)
+	bvs := make(map[uint64]index)
 	// Make the first bitvector positions correspond with the groups.
 	for r, i := range b.groups {
-		bv := uint32(1 << i)
+		bv := uint64(1 << i)
 		for _, g := range mm[r] {
 			bv |= 1 << g
 		}
@@ -1589,7 +1602,7 @@ func (b *builder) writeRegionInclusionData() {
 	}
 	for r := 1; r < len(b.region.s); r++ {
 		if _, ok := b.groups[r]; !ok {
-			bv := uint32(0)
+			bv := uint64(0)
 			for _, g := range mm[r] {
 				bv |= 1 << g
 			}
@@ -1604,9 +1617,9 @@ func (b *builder) writeRegionInclusionData() {
 		}
 	}
 	b.writeSlice("regionInclusion", regionInclusion)
-	regionInclusionBits := make([]uint32, len(bvs))
+	regionInclusionBits := make([]uint64, len(bvs))
 	for k, v := range bvs {
-		regionInclusionBits[v] = uint32(k)
+		regionInclusionBits[v] = uint64(k)
 	}
 	// Add bit vectors for increasingly large distances until a fixed point is reached.
 	regionInclusionNext := []uint8{}

vendor/golang.org/x/text/language/gen_index.go

@@ -49,7 +49,7 @@ func main() {
 	defer func() {
 		buf := &bytes.Buffer{}
 
-		if _, err = w.WriteGo(buf, "language"); err != nil {
+		if _, err = w.WriteGo(buf, "language", ""); err != nil {
 			log.Fatalf("Error formatting file index.go: %v", err)
 		}
 

vendor/golang.org/x/text/language/language.go

@@ -2,105 +2,10 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-//go:generate go run maketables.go gen_common.go -output tables.go
+//go:generate go run gen.go gen_common.go -output tables.go
 //go:generate go run gen_index.go
 
-// Package language implements BCP 47 language tags and related functionality.
-//
-// The Tag type, which is used to represent languages, is agnostic to the
-// meaning of its subtags. Tags are not fully canonicalized to preserve
-// information that may be valuable in certain contexts. As a consequence, two
-// different tags may represent identical languages.
-//
-// Initializing language- or locale-specific components usually consists of
-// two steps. The first step is to select a display language based on the
-// preferred languages of the user and the languages supported by an application.
-// The second step is to create the language-specific services based on
-// this selection. Each is discussed in more details below.
-//
-// Matching preferred against supported languages
-//
-// An application may support various languages. This list is typically limited
-// by the languages for which there exists translations of the user interface.
-// Similarly, a user may provide a list of preferred languages which is limited
-// by the languages understood by this user.
-// An application should use a Matcher to find the best supported language based
-// on the user's preferred list.
-// Matchers are aware of the intricacies of equivalence between languages.
-// The default Matcher implementation takes into account things such as
-// deprecated subtags, legacy tags, and mutual intelligibility between scripts
-// and languages.
-//
-// A Matcher for English, Australian English, Danish, and standard Mandarin can
-// be defined as follows:
-//
-//		var matcher = language.NewMatcher([]language.Tag{
-//			language.English,   // The first language is used as fallback.
-// 			language.MustParse("en-AU"),
-//			language.Danish,
-//			language.Chinese,
-//		})
-//
-// The following code selects the best match for someone speaking Spanish and
-// Norwegian:
-//
-// 		preferred := []language.Tag{ language.Spanish, language.Norwegian }
-//		tag, _, _ := matcher.Match(preferred...)
-//
-// In this case, the best match is Danish, as Danish is sufficiently a match to
-// Norwegian to not have to fall back to the default.
-// See ParseAcceptLanguage on how to handle the Accept-Language HTTP header.
-//
-// Selecting language-specific services
-//
-// One should always use the Tag returned by the Matcher to create an instance
-// of any of the language-specific services provided by the text repository.
-// This prevents the mixing of languages, such as having a different language for
-// messages and display names, as well as improper casing or sorting order for
-// the selected language.
-// Using the returned Tag also allows user-defined settings, such as collation
-// order or numbering system to be transparently passed as options.
-//
-// If you have language-specific data in your application, however, it will in
-// most cases suffice to use the index returned by the matcher to identify
-// the user language.
-// The following loop provides an alternative in case this is not sufficient:
-//
-// 		supported := map[language.Tag]data{
-//			language.English:            enData,
-// 			language.MustParse("en-AU"): enAUData,
-//			language.Danish:             daData,
-//			language.Chinese:            zhData,
-// 		}
-//		tag, _, _ := matcher.Match(preferred...)
-//		for ; tag != language.Und; tag = tag.Parent() {
-//			if v, ok := supported[tag]; ok {
-//				return v
-//			}
-//		}
-// 		return enData // should not reach here
-//
-// Repeatedly taking the Parent of the tag returned by Match will eventually
-// match one of the tags used to initialize the Matcher.
-//
-// Canonicalization
-//
-// By default, only legacy and deprecated tags are converted into their
-// canonical equivalent. All other information is preserved. This approach makes
-// the confidence scores more accurate and allows matchers to distinguish
-// between variants that are otherwise lost.
-//
-// As a consequence, two tags that should be treated as identical according to
-// BCP 47 or CLDR, like "en-Latn" and "en", will be represented differently. The
-// Matchers will handle such distinctions, though, and are aware of the
-// equivalence relations. The CanonType type can be used to alter the
-// canonicalization form.
-//
-// References
-//
-// BCP 47 - Tags for Identifying Languages
-// http://tools.ietf.org/html/bcp47
-package language // import "golang.org/x/text/language"
+package language
 
 // TODO: Remove above NOTE after:
 // - verifying that tables are dropped correctly (most notably matcher tables).
@@ -129,8 +34,15 @@ const (
 // specific language or locale. All language tag values are guaranteed to be
 // well-formed.
 type Tag struct {
-	lang     langID
-	region   regionID
+	lang   langID
+	region regionID
+	// TODO: we will soon run out of positions for script. Idea: instead of
+	// storing lang, region, and script codes, store only the compact index and
+	// have a lookup table from this code to its expansion. This greatly speeds
+	// up table lookup, speed up common variant cases.
+	// This will also immediately free up 3 extra bytes. Also, the pVariant
+	// field can now be moved to the lookup table, as the compact index uniquely
+	// determines the offset of a possible variant.
 	script   scriptID
 	pVariant byte   // offset in str, includes preceding '-'
 	pExt     uint16 // offset of first extension, includes preceding '-'
@@ -387,6 +299,26 @@ func (t Tag) String() string {
 	return string(buf[:t.genCoreBytes(buf[:])])
 }
 
+// MarshalText implements encoding.TextMarshaler.
+func (t Tag) MarshalText() (text []byte, err error) {
+	if t.str != "" {
+		text = append(text, t.str...)
+	} else if t.script == 0 && t.region == 0 {
+		text = append(text, t.lang.String()...)
+	} else {
+		buf := [maxCoreSize]byte{}
+		text = buf[:t.genCoreBytes(buf[:])]
+	}
+	return text, nil
+}
+
+// UnmarshalText implements encoding.TextUnmarshaler.
+func (t *Tag) UnmarshalText(text []byte) error {
+	tag, err := Raw.Parse(string(text))
+	*t = tag
+	return err
+}
+
 // Base returns the base language of the language tag. If the base language is
 // unspecified, an attempt will be made to infer it from the context.
 // It uses a variant of CLDR's Add Likely Subtags algorithm. This is subject to change.

vendor/golang.org/x/text/language/match.go

@@ -6,6 +6,39 @@ package language
 
 import "errors"
 
+// A MatchOption configures a Matcher.
+type MatchOption func(*matcher)
+
+// PreferSameScript will, in the absence of a match, result in the first
+// preferred tag with the same script as a supported tag to match this supported
+// tag. The default is currently true, but this may change in the future.
+func PreferSameScript(preferSame bool) MatchOption {
+	return func(m *matcher) { m.preferSameScript = preferSame }
+}
+
+// TODO(v1.0.0): consider making Matcher a concrete type, instead of interface.
+// There doesn't seem to be too much need for multiple types.
+// Making it a concrete type allows MatchStrings to be a method, which will
+// improve its discoverability.
+
+// MatchStrings parses and matches the given strings until one of them matches
+// the language in the Matcher. A string may be an Accept-Language header as
+// handled by ParseAcceptLanguage. The default language is returned if no
+// other language matched.
+func MatchStrings(m Matcher, lang ...string) (tag Tag, index int) {
+	for _, accept := range lang {
+		desired, _, err := ParseAcceptLanguage(accept)
+		if err != nil {
+			continue
+		}
+		if tag, index, conf := m.Match(desired...); conf != No {
+			return tag, index
+		}
+	}
+	tag, index, _ = m.Match()
+	return
+}
+
 // Matcher is the interface that wraps the Match method.
 //
 // Match returns the best match for any of the given tags, along with
@@ -36,25 +69,45 @@ func Comprehends(speaker, alternative Tag) Confidence {
 // matched tag in t, but is augmented with the Unicode extension ('u')of the
 // corresponding preferred tag. This allows user locale options to be passed
 // transparently.
-func NewMatcher(t []Tag) Matcher {
-	return newMatcher(t)
+func NewMatcher(t []Tag, options ...MatchOption) Matcher {
+	return newMatcher(t, options)
 }
 
 func (m *matcher) Match(want ...Tag) (t Tag, index int, c Confidence) {
 	match, w, c := m.getBest(want...)
-	if match == nil {
-		t = m.default_.tag
-	} else {
+	if match != nil {
 		t, index = match.tag, match.index
+	} else {
+		// TODO: this should be an option
+		t = m.default_.tag
+		if m.preferSameScript {
+		outer:
+			for _, w := range want {
+				script, _ := w.Script()
+				if script.scriptID == 0 {
+					// Don't do anything if there is no script, such as with
+					// private subtags.
+					continue
+				}
+				for i, h := range m.supported {
+					if script.scriptID == h.maxScript {
+						t, index = h.tag, i
+						break outer
+					}
+				}
+			}
+		}
+		// TODO: select first language tag based on script.
+	}
+	if w.region != 0 && t.region != 0 && t.region.contains(w.region) {
+		t, _ = Raw.Compose(t, Region{w.region})
 	}
 	// Copy options from the user-provided tag into the result tag. This is hard
 	// to do after the fact, so we do it here.
-	// TODO: consider also adding in variants that are compatible with the
-	// matched language.
-	// TODO: Add back region if it is non-ambiguous? Or create another tag to
-	// preserve the region?
-	if u, ok := w.Extension('u'); ok {
-		t, _ = Raw.Compose(t, u)
+	// TODO: add in alternative variants to -u-va-.
+	// TODO: add preferred region to -u-rg-.
+	if e := w.Extensions(); len(e) > 0 {
+		t, _ = Raw.Compose(t, e)
 	}
 	return t, index, c
 }
@@ -91,7 +144,7 @@ var ErrMissingLikelyTagsData = errors.New("missing likely tags data")
 
 // addLikelySubtags sets subtags to their most likely value, given the locale.
 // In most cases this means setting fields for unknown values, but in some
-// cases it may alter a value.  It returns a ErrMissingLikelyTagsData error
+// cases it may alter a value.  It returns an ErrMissingLikelyTagsData error
 // if the given locale cannot be expanded.
 func (t Tag) addLikelySubtags() (Tag, error) {
 	id, err := addTags(t)
@@ -300,8 +353,9 @@ func minimizeTags(t Tag) (Tag, error) {
 //        1) compute the match between the two tags.
 //        2) if the match is better than the previous best match, replace it
 //           with the new match. (see next section)
-//     b) if the current best match is above a certain threshold, return this
-//        match without proceeding to the next tag in "desired". [See Note 1]
+//     b) if the current best match is Exact and pin is true the result will be
+//        frozen to the language found thusfar, although better matches may
+//        still be found for the same language.
 //   3) If the best match so far is below a certain threshold, return "default".
 //
 // Ranking:
@@ -350,9 +404,6 @@ func minimizeTags(t Tag) (Tag, error) {
 // found wins.
 //
 // Notes:
-// [1] Note that even if we may not have a perfect match, if a match is above a
-//     certain threshold, it is considered a better match than any other match
-//     to a tag later in the list of preferred language tags.
 // [2] In practice, as matching of Exact is done in a separate phase from
 //     matching the other levels, we reuse the Exact level to mean MaxExact in
 //     the second phase. As a consequence, we only need the levels defined by
@@ -388,16 +439,18 @@ func minimizeTags(t Tag) (Tag, error) {
 
 // matcher keeps a set of supported language tags, indexed by language.
 type matcher struct {
-	default_     *haveTag
-	index        map[langID]*matchHeader
-	passSettings bool
+	default_         *haveTag
+	supported        []*haveTag
+	index            map[langID]*matchHeader
+	passSettings     bool
+	preferSameScript bool
 }
 
 // matchHeader has the lists of tags for exact matches and matches based on
 // maximized and canonicalized tags for a given language.
 type matchHeader struct {
-	exact []*haveTag
-	max   []*haveTag
+	haveTags []*haveTag
+	original bool
 }
 
 // haveTag holds a supported Tag and its maximized script and region. The maximized
@@ -427,7 +480,7 @@ type haveTag struct {
 
 func makeHaveTag(tag Tag, index int) (haveTag, langID) {
 	max := tag
-	if tag.lang != 0 {
+	if tag.lang != 0 || tag.region != 0 || tag.script != 0 {
 		max, _ = max.canonicalize(All)
 		max, _ = addTags(max)
 		max.remakeString()
@@ -452,29 +505,27 @@ func altScript(l langID, s scriptID) scriptID {
 // addIfNew adds a haveTag to the list of tags only if it is a unique tag.
 // Tags that have the same maximized values are linked by index.
 func (h *matchHeader) addIfNew(n haveTag, exact bool) {
+	h.original = h.original || exact
 	// Don't add new exact matches.
-	for _, v := range h.exact {
+	for _, v := range h.haveTags {
 		if v.tag.equalsRest(n.tag) {
 			return
 		}
 	}
-	if exact {
-		h.exact = append(h.exact, &n)
-	}
 	// Allow duplicate maximized tags, but create a linked list to allow quickly
 	// comparing the equivalents and bail out.
-	for i, v := range h.max {
+	for i, v := range h.haveTags {
 		if v.maxScript == n.maxScript &&
 			v.maxRegion == n.maxRegion &&
 			v.tag.variantOrPrivateTagStr() == n.tag.variantOrPrivateTagStr() {
-			for h.max[i].nextMax != 0 {
-				i = int(h.max[i].nextMax)
+			for h.haveTags[i].nextMax != 0 {
+				i = int(h.haveTags[i].nextMax)
 			}
-			h.max[i].nextMax = uint16(len(h.max))
+			h.haveTags[i].nextMax = uint16(len(h.haveTags))
 			break
 		}
 	}
-	h.max = append(h.max, &n)
+	h.haveTags = append(h.haveTags, &n)
 }
 
 // header returns the matchHeader for the given language. It creates one if
@@ -501,9 +552,13 @@ func toConf(d uint8) Confidence {
 // newMatcher builds an index for the given supported tags and returns it as
 // a matcher. It also expands the index by considering various equivalence classes
 // for a given tag.
-func newMatcher(supported []Tag) *matcher {
+func newMatcher(supported []Tag, options []MatchOption) *matcher {
 	m := &matcher{
-		index: make(map[langID]*matchHeader),
+		index:            make(map[langID]*matchHeader),
+		preferSameScript: true,
+	}
+	for _, o := range options {
+		o(m)
 	}
 	if len(supported) == 0 {
 		m.default_ = &haveTag{}
@@ -514,25 +569,28 @@ func newMatcher(supported []Tag) *matcher {
 	for i, tag := range supported {
 		pair, _ := makeHaveTag(tag, i)
 		m.header(tag.lang).addIfNew(pair, true)
+		m.supported = append(m.supported, &pair)
 	}
-	m.default_ = m.header(supported[0].lang).exact[0]
+	m.default_ = m.header(supported[0].lang).haveTags[0]
+	// Keep these in two different loops to support the case that two equivalent
+	// languages are distinguished, such as iw and he.
 	for i, tag := range supported {
 		pair, max := makeHaveTag(tag, i)
 		if max != tag.lang {
-			m.header(max).addIfNew(pair, false)
+			m.header(max).addIfNew(pair, true)
 		}
 	}
 
 	// update is used to add indexes in the map for equivalent languages.
-	// If force is true, the update will also apply to derived entries. To
-	// avoid applying a "transitive closure", use false.
-	update := func(want, have uint16, conf Confidence, force bool) {
+	// update will only add entries to original indexes, thus not computing any
+	// transitive relations.
+	update := func(want, have uint16, conf Confidence) {
 		if hh := m.index[langID(have)]; hh != nil {
-			if !force && len(hh.exact) == 0 {
+			if !hh.original {
 				return
 			}
 			hw := m.header(langID(want))
-			for _, ht := range hh.max {
+			for _, ht := range hh.haveTags {
 				v := *ht
 				if conf < v.conf {
 					v.conf = conf
@@ -541,7 +599,7 @@ func newMatcher(supported []Tag) *matcher {
 				if v.altScript != 0 {
 					v.altScript = altScript(langID(want), v.maxScript)
 				}
-				hw.addIfNew(v, conf == Exact && len(hh.exact) > 0)
+				hw.addIfNew(v, conf == Exact && hh.original)
 			}
 		}
 	}
@@ -549,9 +607,9 @@ func newMatcher(supported []Tag) *matcher {
 	// Add entries for languages with mutual intelligibility as defined by CLDR's
 	// languageMatch data.
 	for _, ml := range matchLang {
-		update(ml.want, ml.have, toConf(ml.distance), false)
+		update(ml.want, ml.have, toConf(ml.distance))
 		if !ml.oneway {
-			update(ml.have, ml.want, toConf(ml.distance), false)
+			update(ml.have, ml.want, toConf(ml.distance))
 		}
 	}
 
@@ -561,10 +619,6 @@ func newMatcher(supported []Tag) *matcher {
 	// (their canonicalization simply substitutes a different language code, but
 	// nothing else), the match confidence is Exact, otherwise it is High.
 	for i, lm := range langAliasMap {
-		if lm.from == _sh {
-			continue
-		}
-
 		// If deprecated codes match and there is no fiddling with the script or
 		// or region, we consider it an exact match.
 		conf := Exact
@@ -572,9 +626,9 @@ func newMatcher(supported []Tag) *matcher {
 			if !isExactEquivalent(langID(lm.from)) {
 				conf = High
 			}
-			update(lm.to, lm.from, conf, true)
+			update(lm.to, lm.from, conf)
 		}
-		update(lm.from, lm.to, conf, true)
+		update(lm.from, lm.to, conf)
 	}
 	return m
 }
@@ -583,28 +637,29 @@ func newMatcher(supported []Tag) *matcher {
 // account the order of preference of the given tags.
 func (m *matcher) getBest(want ...Tag) (got *haveTag, orig Tag, c Confidence) {
 	best := bestMatch{}
-	for _, w := range want {
+	for i, w := range want {
 		var max Tag
 		// Check for exact match first.
 		h := m.index[w.lang]
 		if w.lang != 0 {
-			// Base language is defined.
 			if h == nil {
 				continue
 			}
-			for i := range h.exact {
-				have := h.exact[i]
-				if have.tag.equalsRest(w) {
-					return have, w, Exact
-				}
+			// Base language is defined.
+			max, _ = w.canonicalize(Legacy | Deprecated | Macro)
+			// A region that is added through canonicalization is stronger than
+			// a maximized region: set it in the original (e.g. mo -> ro-MD).
+			if w.region != max.region {
+				w.region = max.region
 			}
-			max, _ = w.canonicalize(Legacy | Deprecated)
+			// TODO: should we do the same for scripts?
+			// See test case: en, sr, nl ; sh ; sr
 			max, _ = addTags(max)
 		} else {
 			// Base language is not defined.
 			if h != nil {
-				for i := range h.exact {
-					have := h.exact[i]
+				for i := range h.haveTags {
+					have := h.haveTags[i]
 					if have.tag.equalsRest(w) {
 						return have, w, Exact
 					}
@@ -620,16 +675,23 @@ func (m *matcher) getBest(want ...Tag) (got *haveTag, orig Tag, c Confidence) {
 				continue
 			}
 		}
+		pin := true
+		for _, t := range want[i+1:] {
+			if w.lang == t.lang {
+				pin = false
+				break
+			}
+		}
 		// Check for match based on maximized tag.
-		for i := range h.max {
-			have := h.max[i]
-			best.update(have, w, max.script, max.region)
+		for i := range h.haveTags {
+			have := h.haveTags[i]
+			best.update(have, w, max.script, max.region, pin)
 			if best.conf == Exact {
 				for have.nextMax != 0 {
-					have = h.max[have.nextMax]
-					best.update(have, w, max.script, max.region)
+					have = h.haveTags[have.nextMax]
+					best.update(have, w, max.script, max.region, pin)
 				}
-				return best.have, best.want, High
+				return best.have, best.want, best.conf
 			}
 		}
 	}
@@ -644,42 +706,68 @@ func (m *matcher) getBest(want ...Tag) (got *haveTag, orig Tag, c Confidence) {
 
 // bestMatch accumulates the best match so far.
 type bestMatch struct {
-	have *haveTag
-	want Tag
-	conf Confidence
+	have            *haveTag
+	want            Tag
+	conf            Confidence
+	pinnedRegion    regionID
+	pinLanguage     bool
+	sameRegionGroup bool
 	// Cached results from applying tie-breaking rules.
-	origLang   bool
-	origReg    bool
-	regDist    uint8
-	origScript bool
-	parentDist uint8 // 255 if have is not an ancestor of want tag.
+	origLang     bool
+	origReg      bool
+	paradigmReg  bool
+	regGroupDist uint8
+	origScript   bool
 }
 
 // update updates the existing best match if the new pair is considered to be a
-// better match.
-// To determine if the given pair is a better match, it first computes the rough
-// confidence level. If this surpasses the current match, it will replace it and
-// update the tie-breaker rule cache. If there is a tie, it proceeds with applying
-// a series of tie-breaker rules. If there is no conclusive winner after applying
-// the tie-breaker rules, it leaves the current match as the preferred match.
-func (m *bestMatch) update(have *haveTag, tag Tag, maxScript scriptID, maxRegion regionID) {
+// better match. To determine if the given pair is a better match, it first
+// computes the rough confidence level. If this surpasses the current match, it
+// will replace it and update the tie-breaker rule cache. If there is a tie, it
+// proceeds with applying a series of tie-breaker rules. If there is no
+// conclusive winner after applying the tie-breaker rules, it leaves the current
+// match as the preferred match.
+//
+// If pin is true and have and tag are a strong match, it will henceforth only
+// consider matches for this language. This corresponds to the nothing that most
+// users have a strong preference for the first defined language. A user can
+// still prefer a second language over a dialect of the preferred language by
+// explicitly specifying dialects, e.g. "en, nl, en-GB". In this case pin should
+// be false.
+func (m *bestMatch) update(have *haveTag, tag Tag, maxScript scriptID, maxRegion regionID, pin bool) {
 	// Bail if the maximum attainable confidence is below that of the current best match.
 	c := have.conf
 	if c < m.conf {
 		return
 	}
-	if have.maxScript != maxScript {
+	// Don't change the language once we already have found an exact match.
+	if m.pinLanguage && tag.lang != m.want.lang {
+		return
+	}
+	// Pin the region group if we are comparing tags for the same language.
+	if tag.lang == m.want.lang && m.sameRegionGroup {
+		_, sameGroup := regionGroupDist(m.pinnedRegion, have.maxRegion, have.maxScript, m.want.lang)
+		if !sameGroup {
+			return
+		}
+	}
+	if c == Exact && have.maxScript == maxScript {
+		// If there is another language and then another entry of this language,
+		// don't pin anything, otherwise pin the language.
+		m.pinLanguage = pin
+	}
+	if have.tag.equalsRest(tag) {
+	} else if have.maxScript != maxScript {
 		// There is usually very little comprehension between different scripts.
-		// In a few cases there may still be Low comprehension. This possibility is
-		// pre-computed and stored in have.altScript.
+		// In a few cases there may still be Low comprehension. This possibility
+		// is pre-computed and stored in have.altScript.
 		if Low < m.conf || have.altScript != maxScript {
 			return
 		}
 		c = Low
 	} else if have.maxRegion != maxRegion {
-		// There is usually a small difference between languages across regions.
-		// We use the region distance (below) to disambiguate between equal matches.
 		if High < c {
+			// There is usually a small difference between languages across regions.
 			c = High
 		}
 	}
@@ -715,10 +803,17 @@ func (m *bestMatch) update(have *haveTag, tag Tag, maxScript scriptID, maxRegion
 		beaten = true
 	}
 
-	// Next we prefer smaller distances between regions, as defined by regionDist.
-	regDist := regionDist(have.maxRegion, maxRegion, tag.lang)
-	if !beaten && m.regDist != regDist {
-		if regDist > m.regDist {
+	regGroupDist, sameGroup := regionGroupDist(have.maxRegion, maxRegion, maxScript, tag.lang)
+	if !beaten && m.regGroupDist != regGroupDist {
+		if regGroupDist > m.regGroupDist {
+			return
+		}
+		beaten = true
+	}
+
+	paradigmReg := isParadigmLocale(tag.lang, have.maxRegion)
+	if !beaten && m.paradigmReg != paradigmReg {
+		if !paradigmReg {
 			return
 		}
 		beaten = true
@@ -733,77 +828,52 @@ func (m *bestMatch) update(have *haveTag, tag Tag, maxScript scriptID, maxRegion
 		beaten = true
 	}
 
-	// Finally we prefer tags which have a closer parent relationship.
-	parentDist := parentDistance(have.tag.region, tag)
-	if !beaten && m.parentDist != parentDist {
-		if parentDist > m.parentDist {
-			return
-		}
-		beaten = true
-	}
-
 	// Update m to the newly found best match.
 	if beaten {
 		m.have = have
 		m.want = tag
 		m.conf = c
+		m.pinnedRegion = maxRegion
+		m.sameRegionGroup = sameGroup
 		m.origLang = origLang
 		m.origReg = origReg
+		m.paradigmReg = paradigmReg
 		m.origScript = origScript
-		m.regDist = regDist
-		m.parentDist = parentDist
+		m.regGroupDist = regGroupDist
 	}
 }
 
-// parentDistance returns the number of times Parent must be called before the
-// regions match. It is assumed that it has already been checked that lang and
-// script are identical. If haveRegion does not occur in the ancestor chain of
-// tag, it returns 255.
-func parentDistance(haveRegion regionID, tag Tag) uint8 {
-	p := tag.Parent()
-	d := uint8(1)
-	for haveRegion != p.region {
-		if p.region == 0 {
-			return 255
-		}
-		p = p.Parent()
-		d++
-	}
-	return d
-}
-
-// regionDist wraps regionDistance with some exceptions to the algorithmic distance.
-func regionDist(a, b regionID, lang langID) uint8 {
-	if lang == _en {
-		// Two variants of non-US English are close to each other, regardless of distance.
-		if a != _US && b != _US {
-			return 2
+func isParadigmLocale(lang langID, r regionID) bool {
+	for _, e := range paradigmLocales {
+		if langID(e[0]) == lang && (r == regionID(e[1]) || r == regionID(e[2])) {
+			return true
 		}
 	}
-	return uint8(regionDistance(a, b))
+	return false
 }
 
-// regionDistance computes the distance between two regions based on the
-// distance in the graph of region containments as defined in CLDR. It iterates
-// over increasingly inclusive sets of groups, represented as bit vectors, until
-// the source bit vector has bits in common with the destination vector.
-func regionDistance(a, b regionID) int {
-	if a == b {
-		return 0
+// regionGroupDist computes the distance between two regions based on their
+// CLDR grouping.
+func regionGroupDist(a, b regionID, script scriptID, lang langID) (dist uint8, same bool) {
+	const defaultDistance = 4
+
+	aGroup := uint(regionToGroups[a]) << 1
+	bGroup := uint(regionToGroups[b]) << 1
+	for _, ri := range matchRegion {
+		if langID(ri.lang) == lang && (ri.script == 0 || scriptID(ri.script) == script) {
+			group := uint(1 << (ri.group &^ 0x80))
+			if 0x80&ri.group == 0 {
+				if aGroup&bGroup&group != 0 { // Both regions are in the group.
+					return ri.distance, ri.distance == defaultDistance
+				}
+			} else {
+				if (aGroup|bGroup)&group == 0 { // Both regions are not in the group.
+					return ri.distance, ri.distance == defaultDistance
+				}
+			}
+		}
 	}
-	p, q := regionInclusion[a], regionInclusion[b]
-	if p < nRegionGroups {
-		p, q = q, p
-	}
-	set := regionInclusionBits
-	if q < nRegionGroups && set[p]&(1<<q) != 0 {
-		return 1
-	}
-	d := 2
-	for goal := set[q]; set[p]&goal == 0; p = regionInclusionNext[p] {
-		d++
-	}
-	return d
+	return defaultDistance, true
 }
 
 func (t Tag) variants() string {
@@ -850,4 +920,14 @@ func init() {
 			notEquivalent = append(notEquivalent, langID(lm.from))
 		}
 	}
+	// Maximize undefined regions of paradigm locales.
+	for i, v := range paradigmLocales {
+		max, _ := addTags(Tag{lang: langID(v[0])})
+		if v[1] == 0 {
+			paradigmLocales[i][1] = uint16(max.region)
+		}
+		if v[2] == 0 {
+			paradigmLocales[i][2] = uint16(max.region)
+		}
+	}
 }

vendor/golang.org/x/text/language/parse.go

@@ -765,7 +765,7 @@ func nextExtension(s string, p int) int {
 
 var errInvalidWeight = errors.New("ParseAcceptLanguage: invalid weight")
 
-// ParseAcceptLanguage parses the contents of a Accept-Language header as
+// ParseAcceptLanguage parses the contents of an Accept-Language header as
 // defined in http://www.ietf.org/rfc/rfc2616.txt and returns a list of Tags and
 // a list of corresponding quality weights. It is more permissive than RFC 2616
 // and may return non-nil slices even if the input is not valid.