Update dependencies to K8s 1.8

vendor/golang.org/x/text/cases/cases.go

@@ -35,7 +35,7 @@ import (
 // A Caser may be stateful and should therefore not be shared between
 // goroutines.
 type Caser struct {
-	t transform.Transformer
+	t transform.SpanningTransformer
 }
 
 // Bytes returns a new byte slice with the result of converting b to the case
@@ -56,12 +56,17 @@ func (c Caser) String(s string) string {
 // Transform.
 func (c Caser) Reset() { c.t.Reset() }
 
-// Transform implements the Transformer interface and transforms the given input
-// to the case form implemented by c.
+// Transform implements the transform.Transformer interface and transforms the
+// given input to the case form implemented by c.
 func (c Caser) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
 	return c.t.Transform(dst, src, atEOF)
 }
 
+// Span implements the transform.SpanningTransformer interface.
+func (c Caser) Span(src []byte, atEOF bool) (n int, err error) {
+	return c.t.Span(src, atEOF)
+}
+
 // Upper returns a Caser for language-specific uppercasing.
 func Upper(t language.Tag, opts ...Option) Caser {
 	return Caser{makeUpper(t, getOpts(opts...))}
@@ -83,14 +88,20 @@ func Title(t language.Tag, opts ...Option) Caser {
 //
 // Case folding does not normalize the input and may not preserve a normal form.
 // Use the collate or search package for more convenient and linguistically
-// sound comparisons.  Use unicode/precis for string comparisons where security
-// aspects are a concern.
+// sound comparisons. Use golang.org/x/text/secure/precis for string comparisons
+// where security aspects are a concern.
 func Fold(opts ...Option) Caser {
 	return Caser{makeFold(getOpts(opts...))}
 }
 
 // An Option is used to modify the behavior of a Caser.
-type Option func(o *options)
+type Option func(o options) options
+
+// TODO: consider these options to take a boolean as well, like FinalSigma.
+// The advantage of using this approach is that other providers of a lower-case
+// algorithm could set different defaults by prefixing a user-provided slice
+// of options with their own. This is handy, for instance, for the precis
+// package which would override the default to not handle the Greek final sigma.
 
 var (
 	// NoLower disables the lowercasing of non-leading letters for a title
@@ -110,20 +121,42 @@ type options struct {
 
 	// TODO: segmenter, max ignorable, alternative versions, etc.
 
-	noFinalSigma bool // Only used for testing.
+	ignoreFinalSigma bool
 }
 
 func getOpts(o ...Option) (res options) {
 	for _, f := range o {
-		f(&res)
+		res = f(res)
 	}
 	return
 }
 
-func noLower(o *options) {
+func noLower(o options) options {
 	o.noLower = true
+	return o
 }
 
-func compact(o *options) {
+func compact(o options) options {
 	o.simple = true
+	return o
+}
+
+// HandleFinalSigma specifies whether the special handling of Greek final sigma
+// should be enabled. Unicode prescribes handling the Greek final sigma for all
+// locales, but standards like IDNA and PRECIS override this default.
+func HandleFinalSigma(enable bool) Option {
+	if enable {
+		return handleFinalSigma
+	}
+	return ignoreFinalSigma
+}
+
+func ignoreFinalSigma(o options) options {
+	o.ignoreFinalSigma = true
+	return o
+}
+
+func handleFinalSigma(o options) options {
+	o.ignoreFinalSigma = false
+	return o
 }

vendor/golang.org/x/text/cases/context.go

@@ -4,9 +4,7 @@
 
 package cases
 
-import (
-	"golang.org/x/text/transform"
-)
+import "golang.org/x/text/transform"
 
 // A context is used for iterating over source bytes, fetching case info and
 // writing to a destination buffer.
@@ -56,6 +54,14 @@ func (c *context) ret() (nDst, nSrc int, err error) {
 	return c.nDst, c.nSrc, transform.ErrShortSrc
 }
 
+// retSpan returns the return values for the Span method. It checks whether
+// there were insufficient bytes in src to complete and introduces an error
+// accordingly, if necessary.
+func (c *context) retSpan() (n int, err error) {
+	_, nSrc, err := c.ret()
+	return nSrc, err
+}
+
 // checkpoint sets the return value buffer points for Transform to the current
 // positions.
 func (c *context) checkpoint() {
@@ -200,6 +206,23 @@ func lower(c *context) bool {
 	return c.copy()
 }
 
+func isLower(c *context) bool {
+	ct := c.caseType()
+	if c.info&hasMappingMask == 0 || ct == cLower {
+		return true
+	}
+	if c.info&exceptionBit == 0 {
+		c.err = transform.ErrEndOfSpan
+		return false
+	}
+	e := exceptions[c.info>>exceptionShift:]
+	if nLower := (e[1] >> lengthBits) & lengthMask; nLower != noChange {
+		c.err = transform.ErrEndOfSpan
+		return false
+	}
+	return true
+}
+
 // upper writes the uppercase version of the current rune to dst.
 func upper(c *context) bool {
 	ct := c.caseType()
@@ -226,6 +249,29 @@ func upper(c *context) bool {
 	return c.copy()
 }
 
+// isUpper writes the isUppercase version of the current rune to dst.
+func isUpper(c *context) bool {
+	ct := c.caseType()
+	if c.info&hasMappingMask == 0 || ct == cUpper {
+		return true
+	}
+	if c.info&exceptionBit == 0 {
+		c.err = transform.ErrEndOfSpan
+		return false
+	}
+	e := exceptions[c.info>>exceptionShift:]
+	// Get length of first special case mapping.
+	n := (e[1] >> lengthBits) & lengthMask
+	if ct == cTitle {
+		n = e[1] & lengthMask
+	}
+	if n != noChange {
+		c.err = transform.ErrEndOfSpan
+		return false
+	}
+	return true
+}
+
 // title writes the title case version of the current rune to dst.
 func title(c *context) bool {
 	ct := c.caseType()
@@ -257,6 +303,33 @@ func title(c *context) bool {
 	return c.copy()
 }
 
+// isTitle reports whether the current rune is in title case.
+func isTitle(c *context) bool {
+	ct := c.caseType()
+	if c.info&hasMappingMask == 0 || ct == cTitle {
+		return true
+	}
+	if c.info&exceptionBit == 0 {
+		if ct == cLower {
+			c.err = transform.ErrEndOfSpan
+			return false
+		}
+		return true
+	}
+	// Get the exception data.
+	e := exceptions[c.info>>exceptionShift:]
+	if nTitle := e[1] & lengthMask; nTitle != noChange {
+		c.err = transform.ErrEndOfSpan
+		return false
+	}
+	nFirst := (e[1] >> lengthBits) & lengthMask
+	if ct == cLower && nFirst != noChange {
+		c.err = transform.ErrEndOfSpan
+		return false
+	}
+	return true
+}
+
 // foldFull writes the foldFull version of the current rune to dst.
 func foldFull(c *context) bool {
 	if c.info&hasMappingMask == 0 {
@@ -279,3 +352,25 @@ func foldFull(c *context) bool {
 	}
 	return c.writeString(e[2 : 2+n])
 }
+
+// isFoldFull reports whether the current run is mapped to foldFull
+func isFoldFull(c *context) bool {
+	if c.info&hasMappingMask == 0 {
+		return true
+	}
+	ct := c.caseType()
+	if c.info&exceptionBit == 0 {
+		if ct != cLower || c.info&inverseFoldBit != 0 {
+			c.err = transform.ErrEndOfSpan
+			return false
+		}
+		return true
+	}
+	e := exceptions[c.info>>exceptionShift:]
+	n := e[0] & lengthMask
+	if n == 0 && ct == cLower {
+		return true
+	}
+	c.err = transform.ErrEndOfSpan
+	return false
+}

vendor/golang.org/x/text/cases/fold.go

@@ -18,7 +18,15 @@ func (t *caseFolder) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err
 	return c.ret()
 }
 
-func makeFold(o options) transform.Transformer {
+func (t *caseFolder) Span(src []byte, atEOF bool) (n int, err error) {
+	c := context{src: src, atEOF: atEOF}
+	for c.next() && isFoldFull(&c) {
+		c.checkpoint()
+	}
+	return c.retSpan()
+}
+
+func makeFold(o options) transform.SpanningTransformer {
 	// TODO: Special case folding, through option Language, Special/Turkic, or
 	// both.
 	// TODO: Implement Compact options.

vendor/golang.org/x/text/cases/gen.go

@@ -76,7 +76,7 @@ type breakCategory int
 const (
 	breakBreak breakCategory = iota
 	breakLetter
-	breakIgnored
+	breakMid
 )
 
 // mapping returns the case mapping for the given case type.
@@ -162,9 +162,14 @@ func parseUCD() []runeInfo {
 
 		// We collapse the word breaking properties onto the categories we need.
 		switch p.String(1) { // TODO: officially we need to canonicalize.
-		case "Format", "MidLetter", "MidNumLet", "Single_Quote":
-			ri.BreakCat = breakIgnored
-		case "ALetter", "Hebrew_Letter", "Numeric", "Extend", "ExtendNumLet":
+		case "MidLetter", "MidNumLet", "Single_Quote":
+			ri.BreakCat = breakMid
+			if !ri.CaseIgnorable {
+				// finalSigma relies on the fact that all breakMid runes are
+				// also a Case_Ignorable. Revisit this code when this changes.
+				log.Fatalf("Rune %U, which has a break category mid, is not a case ignorable", ri)
+			}
+		case "ALetter", "Hebrew_Letter", "Numeric", "Extend", "ExtendNumLet", "Format", "ZWJ":
 			ri.BreakCat = breakLetter
 		}
 	})
@@ -240,8 +245,11 @@ func makeEntry(ri *runeInfo) {
 	case above: // Above
 		ccc = cccAbove
 	}
-	if ri.BreakCat == breakBreak {
+	switch ri.BreakCat {
+	case breakBreak:
 		ccc = cccBreak
+	case breakMid:
+		ri.entry |= isMidBit
 	}
 
 	ri.entry |= ccc
@@ -588,7 +596,7 @@ func verifyProperties(chars []runeInfo) {
 				// decomposition is greater than U+00FF, the rune is always
 				// great and not a modifier.
 				if f := runes[0]; unicode.IsMark(f) || f > 0xFF && !unicode.Is(unicode.Greek, f) {
-					log.Fatalf("%U: expeced first rune of Greek decomposition to be letter, found %U", r, f)
+					log.Fatalf("%U: expected first rune of Greek decomposition to be letter, found %U", r, f)
 				}
 				// A.6.2: Any follow-up rune in a Greek decomposition is a
 				// modifier of which the first should be gobbled in
@@ -597,7 +605,7 @@ func verifyProperties(chars []runeInfo) {
 					switch m {
 					case 0x0313, 0x0314, 0x0301, 0x0300, 0x0306, 0x0342, 0x0308, 0x0304, 0x345:
 					default:
-						log.Fatalf("%U: modifier %U is outside of expeced Greek modifier set", r, m)
+						log.Fatalf("%U: modifier %U is outside of expected Greek modifier set", r, m)
 					}
 				}
 			}
@@ -690,7 +698,7 @@ func genTablesTest() {
 	parse("auxiliary/WordBreakProperty.txt", func(p *ucd.Parser) {
 		switch p.String(1) {
 		case "Extend", "Format", "MidLetter", "MidNumLet", "Single_Quote",
-			"ALetter", "Hebrew_Letter", "Numeric", "ExtendNumLet":
+			"ALetter", "Hebrew_Letter", "Numeric", "ExtendNumLet", "ZWJ":
 			notBreak[p.Rune(0)] = true
 		}
 	})

vendor/golang.org/x/text/cases/gen_trieval.go

@@ -26,6 +26,7 @@ package main
 //            Only 13..8 are used for XOR patterns.
 //         7  inverseFold (fold to upper, not to lower)
 //         6  index: interpret the XOR pattern as an index
+//            or isMid if case mode is cIgnorableUncased.
 //      5..4  CCC: zero (normal or break), above or other
 //   }
 //      3  exception: interpret this value as an exception index
@@ -48,6 +49,7 @@ const (
 	ignorableValue = 0x0004
 
 	inverseFoldBit = 1 << 7
+	isMidBit       = 1 << 6
 
 	exceptionBit     = 1 << 3
 	exceptionShift   = 5
@@ -57,7 +59,7 @@ const (
 	xorShift    = 8
 
 	// There is no mapping if all xor bits and the exception bit are zero.
-	hasMappingMask = 0xffc0 | exceptionBit
+	hasMappingMask = 0xff80 | exceptionBit
 )
 
 // The case mode bits encodes the case type of a rune. This includes uncased,
@@ -95,10 +97,6 @@ func (c info) isCaseIgnorable() bool {
 	return c&ignorableMask == ignorableValue
 }
 
-func (c info) isCaseIgnorableAndNonBreakStarter() bool {
-	return c&(fullCasedMask|cccMask) == (ignorableValue | cccZero)
-}
-
 func (c info) isNotCasedAndNotCaseIgnorable() bool {
 	return c&fullCasedMask == 0
 }
@@ -107,6 +105,10 @@ func (c info) isCaseIgnorableAndNotCased() bool {
 	return c&fullCasedMask == cIgnorableUncased
 }
 
+func (c info) isMid() bool {
+	return c&(fullCasedMask|isMidBit) == isMidBit|cIgnorableUncased
+}
+
 // The case mapping implementation will need to know about various Canonical
 // Combining Class (CCC) values. We encode two of these in the trie value:
 // cccZero (0) and cccAbove (230). If the value is cccOther, it means that

vendor/golang.org/x/text/cases/icu.go

@@ -0,0 +1,61 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build icu
+
+package cases
+
+// Ideally these functions would be defined in a test file, but go test doesn't
+// allow CGO in tests. The build tag should ensure either way that these
+// functions will not end up in the package.
+
+// TODO: Ensure that the correct ICU version is set.
+
+/*
+#cgo LDFLAGS: -licui18n.57 -licuuc.57
+#include <stdlib.h>
+#include <unicode/ustring.h>
+#include <unicode/utypes.h>
+#include <unicode/localpointer.h>
+#include <unicode/ucasemap.h>
+*/
+import "C"
+
+import "unsafe"
+
+func doICU(tag, caser, input string) string {
+	err := C.UErrorCode(0)
+	loc := C.CString(tag)
+	cm := C.ucasemap_open(loc, C.uint32_t(0), &err)
+
+	buf := make([]byte, len(input)*4)
+	dst := (*C.char)(unsafe.Pointer(&buf[0]))
+	src := C.CString(input)
+
+	cn := C.int32_t(0)
+
+	switch caser {
+	case "fold":
+		cn = C.ucasemap_utf8FoldCase(cm,
+			dst, C.int32_t(len(buf)),
+			src, C.int32_t(len(input)),
+			&err)
+	case "lower":
+		cn = C.ucasemap_utf8ToLower(cm,
+			dst, C.int32_t(len(buf)),
+			src, C.int32_t(len(input)),
+			&err)
+	case "upper":
+		cn = C.ucasemap_utf8ToUpper(cm,
+			dst, C.int32_t(len(buf)),
+			src, C.int32_t(len(input)),
+			&err)
+	case "title":
+		cn = C.ucasemap_utf8ToTitle(cm,
+			dst, C.int32_t(len(buf)),
+			src, C.int32_t(len(input)),
+			&err)
+	}
+	return string(buf[:cn])
+}

vendor/golang.org/x/text/cases/info.go

@@ -28,9 +28,6 @@ func (c info) cccType() info {
 // only makes sense, though, if the performance and/or space penalty of using
 // the generic breaker is big. Extra data will only be needed for non-cased
 // runes, which means there are sufficient bits left in the caseType.
-// Also note that the standard breaking algorithm doesn't always make sense
-// for title casing. For example, a4a -> A4a, but a"4a -> A"4A (where " stands
-// for modifier \u0308).
 // ICU prohibits breaking in such cases as well.
 
 // For the purpose of title casing we use an approximation of the Unicode Word
@@ -41,17 +38,19 @@ func (c info) cccType() info {
 // categories, with associated rules:
 //
 // 1) Letter:
-//    ALetter, Hebrew_Letter, Numeric, ExtendNumLet, Extend.
+//    ALetter, Hebrew_Letter, Numeric, ExtendNumLet, Extend, Format_FE, ZWJ.
 //    Rule: Never break between consecutive runes of this category.
 //
 // 2) Mid:
-//    Format, MidLetter, MidNumLet, Single_Quote.
-//    (Cf. case-ignorable: MidLetter, MidNumLet or cat is Mn, Me, Cf, Lm or Sk).
+//    MidLetter, MidNumLet, Single_Quote.
+//    (Cf. case-ignorable: MidLetter, MidNumLet, Single_Quote or cat is Mn,
+//    Me, Cf, Lm or Sk).
 //    Rule: Don't break between Letter and Mid, but break between two Mids.
 //
 // 3) Break:
-//    Any other category, including NewLine, CR, LF and Double_Quote. These
-//    categories should always result in a break between two cased letters.
+//    Any other category: NewLine, MidNum, CR, LF, Double_Quote, Katakana, and
+//    Other.
+//    These categories should always result in a break between two cased letters.
 //    Rule: Always break.
 //
 // Note 1: the Katakana and MidNum categories can, in esoteric cases, result in

vendor/golang.org/x/text/cases/map.go

@@ -13,6 +13,7 @@ import (
 	"unicode"
 	"unicode/utf8"
 
+	"golang.org/x/text/internal"
 	"golang.org/x/text/language"
 	"golang.org/x/text/transform"
 	"golang.org/x/text/unicode/norm"
@@ -24,6 +25,11 @@ import (
 // dst so far won't need changing as we see more source bytes.
 type mapFunc func(*context) bool
 
+// A spanFunc takes a context set to the current rune and returns whether this
+// rune would be altered when written to the output. It may advance the context
+// to the next rune. It returns whether a checkpoint is possible.
+type spanFunc func(*context) bool
+
 // maxIgnorable defines the maximum number of ignorables to consider for
 // lookahead operations.
 const maxIgnorable = 30
@@ -36,12 +42,12 @@ func init() {
 	for _, s := range strings.Split(supported, " ") {
 		tags = append(tags, language.MustParse(s))
 	}
-	matcher = language.NewMatcher(tags)
+	matcher = internal.NewInheritanceMatcher(tags)
 	Supported = language.NewCoverage(tags)
 }
 
 var (
-	matcher language.Matcher
+	matcher *internal.InheritanceMatcher
 
 	Supported language.Coverage
 
@@ -50,56 +56,69 @@ var (
 
 	// Some uppercase mappers are stateless, so we can precompute the
 	// Transformers and save a bit on runtime allocations.
-	upperFunc = []mapFunc{
-		nil,              // und
-		nil,              // af
-		aztrUpper(upper), // az
-		elUpper,          // el
-		ltUpper(upper),   // lt
-		nil,              // nl
-		aztrUpper(upper), // tr
+	upperFunc = []struct {
+		upper mapFunc
+		span  spanFunc
+	}{
+		{nil, nil},                  // und
+		{nil, nil},                  // af
+		{aztrUpper(upper), isUpper}, // az
+		{elUpper, noSpan},           // el
+		{ltUpper(upper), noSpan},    // lt
+		{nil, nil},                  // nl
+		{aztrUpper(upper), isUpper}, // tr
 	}
 
-	undUpper transform.Transformer = &undUpperCaser{}
+	undUpper            transform.SpanningTransformer = &undUpperCaser{}
+	undLower            transform.SpanningTransformer = &undLowerCaser{}
+	undLowerIgnoreSigma transform.SpanningTransformer = &undLowerIgnoreSigmaCaser{}
 
 	lowerFunc = []mapFunc{
-		lower,     // und
-		lower,     // af
+		nil,       // und
+		nil,       // af
 		aztrLower, // az
-		lower,     // el
+		nil,       // el
 		ltLower,   // lt
-		lower,     // nl
+		nil,       // nl
 		aztrLower, // tr
 	}
 
 	titleInfos = []struct {
-		title, lower mapFunc
-		rewrite      func(*context)
+		title     mapFunc
+		lower     mapFunc
+		titleSpan spanFunc
+		rewrite   func(*context)
 	}{
-		{title, lower, nil},                // und
-		{title, lower, afnlRewrite},        // af
-		{aztrUpper(title), aztrLower, nil}, // az
-		{title, lower, nil},                // el
-		{ltUpper(title), ltLower, nil},     // lt
-		{nlTitle, lower, afnlRewrite},      // nl
-		{aztrUpper(title), aztrLower, nil}, // tr
+		{title, lower, isTitle, nil},                // und
+		{title, lower, isTitle, afnlRewrite},        // af
+		{aztrUpper(title), aztrLower, isTitle, nil}, // az
+		{title, lower, isTitle, nil},                // el
+		{ltUpper(title), ltLower, noSpan, nil},      // lt
+		{nlTitle, lower, nlTitleSpan, afnlRewrite},  // nl
+		{aztrUpper(title), aztrLower, isTitle, nil}, // tr
 	}
 )
 
-func makeUpper(t language.Tag, o options) transform.Transformer {
+func makeUpper(t language.Tag, o options) transform.SpanningTransformer {
 	_, i, _ := matcher.Match(t)
-	f := upperFunc[i]
+	f := upperFunc[i].upper
 	if f == nil {
 		return undUpper
 	}
-	return &simpleCaser{f: f}
+	return &simpleCaser{f: f, span: upperFunc[i].span}
 }
 
-func makeLower(t language.Tag, o options) transform.Transformer {
+func makeLower(t language.Tag, o options) transform.SpanningTransformer {
 	_, i, _ := matcher.Match(t)
 	f := lowerFunc[i]
-	if o.noFinalSigma {
-		return &simpleCaser{f: f}
+	if f == nil {
+		if o.ignoreFinalSigma {
+			return undLowerIgnoreSigma
+		}
+		return undLower
+	}
+	if o.ignoreFinalSigma {
+		return &simpleCaser{f: f, span: isLower}
 	}
 	return &lowerCaser{
 		first:   f,
@@ -107,22 +126,28 @@ func makeLower(t language.Tag, o options) transform.Transformer {
 	}
 }
 
-func makeTitle(t language.Tag, o options) transform.Transformer {
+func makeTitle(t language.Tag, o options) transform.SpanningTransformer {
 	_, i, _ := matcher.Match(t)
 	x := &titleInfos[i]
 	lower := x.lower
 	if o.noLower {
 		lower = (*context).copy
-	} else if !o.noFinalSigma {
+	} else if !o.ignoreFinalSigma {
 		lower = finalSigma(lower)
 	}
 	return &titleCaser{
-		title:   x.title,
-		lower:   lower,
-		rewrite: x.rewrite,
+		title:     x.title,
+		lower:     lower,
+		titleSpan: x.titleSpan,
+		rewrite:   x.rewrite,
 	}
 }
 
+func noSpan(c *context) bool {
+	c.err = transform.ErrEndOfSpan
+	return false
+}
+
 // TODO: consider a similar special case for the fast majority lower case. This
 // is a bit more involved so will require some more precise benchmarking to
 // justify it.
@@ -132,7 +157,7 @@ type undUpperCaser struct{ transform.NopResetter }
 // undUpperCaser implements the Transformer interface for doing an upper case
 // mapping for the root locale (und). It eliminates the need for an allocation
 // as it prevents escaping by not using function pointers.
-func (t *undUpperCaser) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
+func (t undUpperCaser) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
 	c := context{dst: dst, src: src, atEOF: atEOF}
 	for c.next() {
 		upper(&c)
@@ -141,26 +166,117 @@ func (t *undUpperCaser) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int,
 	return c.ret()
 }
 
+func (t undUpperCaser) Span(src []byte, atEOF bool) (n int, err error) {
+	c := context{src: src, atEOF: atEOF}
+	for c.next() && isUpper(&c) {
+		c.checkpoint()
+	}
+	return c.retSpan()
+}
+
+// undLowerIgnoreSigmaCaser implements the Transformer interface for doing
+// a lower case mapping for the root locale (und) ignoring final sigma
+// handling. This casing algorithm is used in some performance-critical packages
+// like secure/precis and x/net/http/idna, which warrants its special-casing.
+type undLowerIgnoreSigmaCaser struct{ transform.NopResetter }
+
+func (t undLowerIgnoreSigmaCaser) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
+	c := context{dst: dst, src: src, atEOF: atEOF}
+	for c.next() && lower(&c) {
+		c.checkpoint()
+	}
+	return c.ret()
+
+}
+
+// Span implements a generic lower-casing. This is possible as isLower works
+// for all lowercasing variants. All lowercase variants only vary in how they
+// transform a non-lowercase letter. They will never change an already lowercase
+// letter. In addition, there is no state.
+func (t undLowerIgnoreSigmaCaser) Span(src []byte, atEOF bool) (n int, err error) {
+	c := context{src: src, atEOF: atEOF}
+	for c.next() && isLower(&c) {
+		c.checkpoint()
+	}
+	return c.retSpan()
+}
+
 type simpleCaser struct {
 	context
-	f mapFunc
+	f    mapFunc
+	span spanFunc
 }
 
 // simpleCaser implements the Transformer interface for doing a case operation
 // on a rune-by-rune basis.
 func (t *simpleCaser) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
-	t.context = context{dst: dst, src: src, atEOF: atEOF}
-	c := &t.context
-	for c.next() && t.f(c) {
+	c := context{dst: dst, src: src, atEOF: atEOF}
+	for c.next() && t.f(&c) {
 		c.checkpoint()
 	}
 	return c.ret()
 }
 
+func (t *simpleCaser) Span(src []byte, atEOF bool) (n int, err error) {
+	c := context{src: src, atEOF: atEOF}
+	for c.next() && t.span(&c) {
+		c.checkpoint()
+	}
+	return c.retSpan()
+}
+
+// undLowerCaser implements the Transformer interface for doing a lower case
+// mapping for the root locale (und) ignoring final sigma handling. This casing
+// algorithm is used in some performance-critical packages like secure/precis
+// and x/net/http/idna, which warrants its special-casing.
+type undLowerCaser struct{ transform.NopResetter }
+
+func (t undLowerCaser) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
+	c := context{dst: dst, src: src, atEOF: atEOF}
+
+	for isInterWord := true; c.next(); {
+		if isInterWord {
+			if c.info.isCased() {
+				if !lower(&c) {
+					break
+				}
+				isInterWord = false
+			} else if !c.copy() {
+				break
+			}
+		} else {
+			if c.info.isNotCasedAndNotCaseIgnorable() {
+				if !c.copy() {
+					break
+				}
+				isInterWord = true
+			} else if !c.hasPrefix("Σ") {
+				if !lower(&c) {
+					break
+				}
+			} else if !finalSigmaBody(&c) {
+				break
+			}
+		}
+		c.checkpoint()
+	}
+	return c.ret()
+}
+
+func (t undLowerCaser) Span(src []byte, atEOF bool) (n int, err error) {
+	c := context{src: src, atEOF: atEOF}
+	for c.next() && isLower(&c) {
+		c.checkpoint()
+	}
+	return c.retSpan()
+}
+
 // lowerCaser implements the Transformer interface. The default Unicode lower
 // casing requires different treatment for the first and subsequent characters
 // of a word, most notably to handle the Greek final Sigma.
 type lowerCaser struct {
+	undLowerIgnoreSigmaCaser
+
 	context
 
 	first, midWord mapFunc
@@ -202,7 +318,9 @@ type titleCaser struct {
 	context
 
 	// rune mappings used by the actual casing algorithms.
-	title, lower mapFunc
+	title     mapFunc
+	lower     mapFunc
+	titleSpan spanFunc
 
 	rewrite func(*context)
 }
@@ -228,10 +346,10 @@ func (t *titleCaser) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err
 			t.rewrite(c)
 		}
 
-		wasMid := p.isCaseIgnorableAndNonBreakStarter()
+		wasMid := p.isMid()
 		// Break out of this loop on failure to ensure we do not modify the
 		// state incorrectly.
-		if p.isCased() && !p.isCaseIgnorableAndNotCased() {
+		if p.isCased() {
 			if !c.isMidWord {
 				if !t.title(c) {
 					break
@@ -242,71 +360,139 @@ func (t *titleCaser) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err
 			}
 		} else if !c.copy() {
 			break
-		}
-
-		// TODO: make this an "else if" if we can prove that no rune that does
-		// not match the first condition of the if statement can be a break.
-		if p.isBreak() {
+		} else if p.isBreak() {
 			c.isMidWord = false
 		}
 
 		// As we save the state of the transformer, it is safe to call
 		// checkpoint after any successful write.
-		c.checkpoint()
+		if !(c.isMidWord && wasMid) {
+			c.checkpoint()
+		}
 
 		if !c.next() {
 			break
 		}
-		if wasMid && c.info.isCaseIgnorableAndNonBreakStarter() {
+		if wasMid && c.info.isMid() {
 			c.isMidWord = false
 		}
 	}
 	return c.ret()
 }
 
+func (t *titleCaser) Span(src []byte, atEOF bool) (n int, err error) {
+	t.context = context{src: src, atEOF: atEOF, isMidWord: t.isMidWord}
+	c := &t.context
+
+	if !c.next() {
+		return c.retSpan()
+	}
+
+	for {
+		p := c.info
+		if t.rewrite != nil {
+			t.rewrite(c)
+		}
+
+		wasMid := p.isMid()
+		// Break out of this loop on failure to ensure we do not modify the
+		// state incorrectly.
+		if p.isCased() {
+			if !c.isMidWord {
+				if !t.titleSpan(c) {
+					break
+				}
+				c.isMidWord = true
+			} else if !isLower(c) {
+				break
+			}
+		} else if p.isBreak() {
+			c.isMidWord = false
+		}
+		// As we save the state of the transformer, it is safe to call
+		// checkpoint after any successful write.
+		if !(c.isMidWord && wasMid) {
+			c.checkpoint()
+		}
+
+		if !c.next() {
+			break
+		}
+		if wasMid && c.info.isMid() {
+			c.isMidWord = false
+		}
+	}
+	return c.retSpan()
+}
+
 // finalSigma adds Greek final Sigma handing to another casing function. It
 // determines whether a lowercased sigma should be σ or ς, by looking ahead for
 // case-ignorables and a cased letters.
 func finalSigma(f mapFunc) mapFunc {
 	return func(c *context) bool {
-		// ::NFD();
-		// # 03A3; 03C2; 03A3; 03A3; Final_Sigma; # GREEK CAPITAL LETTER SIGMA
-		// Σ } [:case-ignorable:]* [:cased:] → σ;
-		// [:cased:] [:case-ignorable:]* { Σ → ς;
-		// ::Any-Lower;
-		// ::NFC();
-
 		if !c.hasPrefix("Σ") {
 			return f(c)
 		}
-
-		p := c.pDst
-		c.writeString("ς")
-		// We need to do one more iteration after maxIgnorable, as a cased
-		// letter is not an ignorable and may modify the result.
-		for i := 0; i < maxIgnorable+1; i++ {
-			if !c.next() {
-				return false
-			}
-			if !c.info.isCaseIgnorable() {
-				if c.info.isCased() {
-					// p+1 is guaranteed to be in bounds: if writing ς was
-					// successful, p+1 will contain the second byte of ς. If not,
-					// this function will have returned after c.next returned false.
-					c.dst[p+1]++ // ς → σ
-				}
-				c.unreadRune()
-				return true
-			}
-			// A case ignorable may also introduce a word break, so we may need
-			// to continue searching even after detecting a break.
-			c.isMidWord = c.isMidWord && !c.info.isBreak()
-			c.copy()
-		}
-		return true
+		return finalSigmaBody(c)
 	}
 }
 
+func finalSigmaBody(c *context) bool {
+	// Current rune must be ∑.
+
+	// ::NFD();
+	// # 03A3; 03C2; 03A3; 03A3; Final_Sigma; # GREEK CAPITAL LETTER SIGMA
+	// Σ } [:case-ignorable:]* [:cased:] → σ;
+	// [:cased:] [:case-ignorable:]* { Σ → ς;
+	// ::Any-Lower;
+	// ::NFC();
+
+	p := c.pDst
+	c.writeString("ς")
+
+	// TODO: we should do this here, but right now this will never have an
+	// effect as this is called when the prefix is Sigma, whereas Dutch and
+	// Afrikaans only test for an apostrophe.
+	//
+	// if t.rewrite != nil {
+	// 	t.rewrite(c)
+	// }
+
+	// We need to do one more iteration after maxIgnorable, as a cased
+	// letter is not an ignorable and may modify the result.
+	wasMid := false
+	for i := 0; i < maxIgnorable+1; i++ {
+		if !c.next() {
+			return false
+		}
+		if !c.info.isCaseIgnorable() {
+			// All Midword runes are also case ignorable, so we are
+			// guaranteed to have a letter or word break here. As we are
+			// unreading the run, there is no need to unset c.isMidWord;
+			// the title caser will handle this.
+			if c.info.isCased() {
+				// p+1 is guaranteed to be in bounds: if writing ς was
+				// successful, p+1 will contain the second byte of ς. If not,
+				// this function will have returned after c.next returned false.
+				c.dst[p+1]++ // ς → σ
+			}
+			c.unreadRune()
+			return true
+		}
+		// A case ignorable may also introduce a word break, so we may need
+		// to continue searching even after detecting a break.
+		isMid := c.info.isMid()
+		if (wasMid && isMid) || c.info.isBreak() {
+			c.isMidWord = false
+		}
+		wasMid = isMid
+		c.copy()
+	}
+	return true
+}
+
+// finalSigmaSpan would be the same as isLower.
+
 // elUpper implements Greek upper casing, which entails removing a predefined
 // set of non-blocked modifiers. Note that these accents should not be removed
 // for title casing!
@@ -376,6 +562,8 @@ func elUpper(c *context) bool {
 	return i == maxIgnorable
 }
 
+// TODO: implement elUpperSpan (low-priority: complex and infrequent).
+
 func ltLower(c *context) bool {
 	// From CLDR:
 	// # Introduce an explicit dot above when lowercasing capital I's and J's
@@ -390,10 +578,10 @@ func ltLower(c *context) bool {
 	// ::NFD();
 	// I } [^[:ccc=Not_Reordered:][:ccc=Above:]]* [:ccc=Above:] → i \u0307;
 	// J } [^[:ccc=Not_Reordered:][:ccc=Above:]]* [:ccc=Above:] → j \u0307;
-	// Į } [^[:ccc=Not_Reordered:][:ccc=Above:]]* [:ccc=Above:] → į \u0307;
-	// Ì → i \u0307 \u0300;
-	// Í → i \u0307 \u0301;
-	// Ĩ → i \u0307 \u0303;
+	// I \u0328 (Į) } [^[:ccc=Not_Reordered:][:ccc=Above:]]* [:ccc=Above:] → i \u0328 \u0307;
+	// I \u0300 (Ì) → i \u0307 \u0300;
+	// I \u0301 (Í) → i \u0307 \u0301;
+	// I \u0303 (Ĩ) → i \u0307 \u0303;
 	// ::Any-Lower();
 	// ::NFC();
 
@@ -445,9 +633,16 @@ func ltLower(c *context) bool {
 	return i == maxIgnorable
 }
 
+// ltLowerSpan would be the same as isLower.
+
 func ltUpper(f mapFunc) mapFunc {
 	return func(c *context) bool {
+		// Unicode:
+		// 0307; 0307; ; ; lt After_Soft_Dotted; # COMBINING DOT ABOVE
+		//
 		// From CLDR:
+		// # Remove \u0307 following soft-dotteds (i, j, and the like), with possible
+		// # intervening non-230 marks.
 		// ::NFD();
 		// [:Soft_Dotted:] [^[:ccc=Not_Reordered:][:ccc=Above:]]* { \u0307 → ;
 		// ::Any-Upper();
@@ -511,6 +706,8 @@ func ltUpper(f mapFunc) mapFunc {
 	}
 }
 
+// TODO: implement ltUpperSpan (low priority: complex and infrequent).
+
 func aztrUpper(f mapFunc) mapFunc {
 	return func(c *context) bool {
 		// i→İ;
@@ -571,6 +768,8 @@ Loop:
 	return c.writeString("ı") && c.writeBytes(c.src[start:c.pSrc+c.sz]) && done
 }
 
+// aztrLowerSpan would be the same as isLower.
+
 func nlTitle(c *context) bool {
 	// From CLDR:
 	// # Special titlecasing for Dutch initial "ij".
@@ -591,6 +790,24 @@ func nlTitle(c *context) bool {
 	return true
 }
 
+func nlTitleSpan(c *context) bool {
+	// From CLDR:
+	// # Special titlecasing for Dutch initial "ij".
+	// ::Any-Title();
+	// # Fix up Ij at the beginning of a "word" (per Any-Title, notUAX #29)
+	// [:^WB=ALetter:] [:WB=Extend:]* [[:WB=MidLetter:][:WB=MidNumLet:]]? { Ij } → IJ ;
+	if c.src[c.pSrc] != 'I' {
+		return isTitle(c)
+	}
+	if !c.next() || c.src[c.pSrc] == 'j' {
+		return false
+	}
+	if c.src[c.pSrc] != 'J' {
+		c.unreadRune()
+	}
+	return true
+}
+
 // Not part of CLDR, but see http://unicode.org/cldr/trac/ticket/7078.
 func afnlRewrite(c *context) {
 	if c.hasPrefix("'") || c.hasPrefix("’") {

vendor/golang.org/x/text/cases/trieval.go

@@ -1,4 +1,4 @@
-// This file was generated by go generate; DO NOT EDIT
+// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
 
 package cases
 
@@ -22,6 +22,7 @@ package cases
 //            Only 13..8 are used for XOR patterns.
 //         7  inverseFold (fold to upper, not to lower)
 //         6  index: interpret the XOR pattern as an index
+//            or isMid if case mode is cIgnorableUncased.
 //      5..4  CCC: zero (normal or break), above or other
 //   }
 //      3  exception: interpret this value as an exception index
@@ -44,6 +45,7 @@ const (
 	ignorableValue = 0x0004
 
 	inverseFoldBit = 1 << 7
+	isMidBit       = 1 << 6
 
 	exceptionBit     = 1 << 3
 	exceptionShift   = 5
@@ -53,7 +55,7 @@ const (
 	xorShift    = 8
 
 	// There is no mapping if all xor bits and the exception bit are zero.
-	hasMappingMask = 0xffc0 | exceptionBit
+	hasMappingMask = 0xff80 | exceptionBit
 )
 
 // The case mode bits encodes the case type of a rune. This includes uncased,
@@ -91,10 +93,6 @@ func (c info) isCaseIgnorable() bool {
 	return c&ignorableMask == ignorableValue
 }
 
-func (c info) isCaseIgnorableAndNonBreakStarter() bool {
-	return c&(fullCasedMask|cccMask) == (ignorableValue | cccZero)
-}
-
 func (c info) isNotCasedAndNotCaseIgnorable() bool {
 	return c&fullCasedMask == 0
 }
@@ -103,6 +101,10 @@ func (c info) isCaseIgnorableAndNotCased() bool {
 	return c&fullCasedMask == cIgnorableUncased
 }
 
+func (c info) isMid() bool {
+	return c&(fullCasedMask|isMidBit) == isMidBit|cIgnorableUncased
+}
+
 // The case mapping implementation will need to know about various Canonical
 // Combining Class (CCC) values. We encode two of these in the trie value:
 // cccZero (0) and cccAbove (230). If the value is cccOther, it means that

vendor/golang.org/x/text/internal/gen.go

@@ -0,0 +1,52 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build ignore
+
+package main
+
+import (
+	"log"
+
+	"golang.org/x/text/internal/gen"
+	"golang.org/x/text/language"
+	"golang.org/x/text/unicode/cldr"
+)
+
+func main() {
+	r := gen.OpenCLDRCoreZip()
+	defer r.Close()
+
+	d := &cldr.Decoder{}
+	data, err := d.DecodeZip(r)
+	if err != nil {
+		log.Fatalf("DecodeZip: %v", err)
+	}
+
+	w := gen.NewCodeWriter()
+	defer w.WriteGoFile("tables.go", "internal")
+
+	// Create parents table.
+	parents := make([]uint16, language.NumCompactTags)
+	for _, loc := range data.Locales() {
+		tag := language.MustParse(loc)
+		index, ok := language.CompactIndex(tag)
+		if !ok {
+			continue
+		}
+		parentIndex := 0 // und
+		for p := tag.Parent(); p != language.Und; p = p.Parent() {
+			if x, ok := language.CompactIndex(p); ok {
+				parentIndex = x
+				break
+			}
+		}
+		parents[index] = uint16(parentIndex)
+	}
+
+	w.WriteComment(`
+	Parent maps a compact index of a tag to the compact index of the parent of
+	this tag.`)
+	w.WriteVar("Parent", parents)
+}

vendor/golang.org/x/text/internal/internal.go

@@ -0,0 +1,51 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:generate go run gen.go
+
+// Package internal contains non-exported functionality that are used by
+// packages in the text repository.
+package internal
+
+import (
+	"sort"
+
+	"golang.org/x/text/language"
+)
+
+// SortTags sorts tags in place.
+func SortTags(tags []language.Tag) {
+	sort.Sort(sorter(tags))
+}
+
+type sorter []language.Tag
+
+func (s sorter) Len() int {
+	return len(s)
+}
+
+func (s sorter) Swap(i, j int) {
+	s[i], s[j] = s[j], s[i]
+}
+
+func (s sorter) Less(i, j int) bool {
+	return s[i].String() < s[j].String()
+}
+
+// UniqueTags sorts and filters duplicate tags in place and returns a slice with
+// only unique tags.
+func UniqueTags(tags []language.Tag) []language.Tag {
+	if len(tags) <= 1 {
+		return tags
+	}
+	SortTags(tags)
+	k := 0
+	for i := 1; i < len(tags); i++ {
+		if tags[k].String() < tags[i].String() {
+			k++
+			tags[k] = tags[i]
+		}
+	}
+	return tags[:k+1]
+}

vendor/golang.org/x/text/internal/match.go

@@ -0,0 +1,67 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package internal
+
+// This file contains matchers that implement CLDR inheritance.
+//
+//     See http://unicode.org/reports/tr35/#Locale_Inheritance.
+//
+// Some of the inheritance described in this document is already handled by
+// the cldr package.
+
+import (
+	"golang.org/x/text/language"
+)
+
+// TODO: consider if (some of the) matching algorithm needs to be public after
+// getting some feel about what is generic and what is specific.
+
+// NewInheritanceMatcher returns a matcher that matches based on the inheritance
+// chain.
+//
+// The matcher uses canonicalization and the parent relationship to find a
+// match. The resulting match will always be either Und or a language with the
+// same language and script as the requested language. It will not match
+// languages for which there is understood to be mutual or one-directional
+// intelligibility.
+//
+// A Match will indicate an Exact match if the language matches after
+// canonicalization and High if the matched tag is a parent.
+func NewInheritanceMatcher(t []language.Tag) *InheritanceMatcher {
+	tags := &InheritanceMatcher{make(map[language.Tag]int)}
+	for i, tag := range t {
+		ct, err := language.All.Canonicalize(tag)
+		if err != nil {
+			ct = tag
+		}
+		tags.index[ct] = i
+	}
+	return tags
+}
+
+type InheritanceMatcher struct {
+	index map[language.Tag]int
+}
+
+func (m InheritanceMatcher) Match(want ...language.Tag) (language.Tag, int, language.Confidence) {
+	for _, t := range want {
+		ct, err := language.All.Canonicalize(t)
+		if err != nil {
+			ct = t
+		}
+		conf := language.Exact
+		for {
+			if index, ok := m.index[ct]; ok {
+				return ct, index, conf
+			}
+			if ct == language.Und {
+				break
+			}
+			ct = ct.Parent()
+			conf = language.High
+		}
+	}
+	return language.Und, 0, language.No
+}

vendor/golang.org/x/text/internal/tables.go

@@ -0,0 +1,117 @@
+// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
+
+package internal
+
+// Parent maps a compact index of a tag to the compact index of the parent of
+// this tag.
+var Parent = []uint16{ // 754 elements
+	// Entry 0 - 3F
+	0x0000, 0x0053, 0x00e5, 0x0000, 0x0003, 0x0003, 0x0000, 0x0006,
+	0x0000, 0x0008, 0x0000, 0x000a, 0x0000, 0x000c, 0x000c, 0x000c,
+	0x000c, 0x000c, 0x000c, 0x000c, 0x000c, 0x000c, 0x000c, 0x000c,
+	0x000c, 0x000c, 0x000c, 0x000c, 0x000c, 0x000c, 0x000c, 0x000c,
+	0x000c, 0x000c, 0x000c, 0x000c, 0x000c, 0x000c, 0x000c, 0x000c,
+	0x000c, 0x0000, 0x0000, 0x002a, 0x0000, 0x002c, 0x0000, 0x002e,
+	0x0000, 0x0000, 0x0031, 0x0030, 0x0030, 0x0000, 0x0035, 0x0000,
+	0x0037, 0x0000, 0x0039, 0x0000, 0x003b, 0x0000, 0x003d, 0x0000,
+	// Entry 40 - 7F
+	0x0000, 0x0040, 0x0000, 0x0042, 0x0042, 0x0000, 0x0045, 0x0045,
+	0x0000, 0x0048, 0x0000, 0x004a, 0x0000, 0x0000, 0x004d, 0x004c,
+	0x004c, 0x0000, 0x0051, 0x0051, 0x0051, 0x0051, 0x0000, 0x0056,
+	0x0000, 0x0058, 0x0000, 0x005a, 0x0000, 0x005c, 0x005c, 0x0000,
+	0x005f, 0x0000, 0x0061, 0x0000, 0x0063, 0x0000, 0x0065, 0x0065,
+	0x0000, 0x0068, 0x0000, 0x006a, 0x006a, 0x006a, 0x006a, 0x006a,
+	0x006a, 0x006a, 0x0000, 0x0072, 0x0000, 0x0074, 0x0000, 0x0076,
+	0x0000, 0x0000, 0x0079, 0x0000, 0x007b, 0x0000, 0x007d, 0x0000,
+	// Entry 80 - BF
+	0x007f, 0x007f, 0x0000, 0x0082, 0x0082, 0x0000, 0x0085, 0x0086,
+	0x0086, 0x0086, 0x0085, 0x0087, 0x0086, 0x0086, 0x0086, 0x0085,
+	0x0086, 0x0086, 0x0086, 0x0086, 0x0086, 0x0086, 0x0087, 0x0086,
+	0x0086, 0x0086, 0x0086, 0x0087, 0x0086, 0x0087, 0x0086, 0x0086,
+	0x0087, 0x0086, 0x0086, 0x0086, 0x0086, 0x0086, 0x0086, 0x0086,
+	0x0086, 0x0086, 0x0085, 0x0086, 0x0086, 0x0086, 0x0086, 0x0086,
+	0x0086, 0x0086, 0x0086, 0x0086, 0x0086, 0x0086, 0x0086, 0x0086,
+	0x0086, 0x0086, 0x0086, 0x0086, 0x0085, 0x0086, 0x0085, 0x0086,
+	// Entry C0 - FF
+	0x0086, 0x0086, 0x0086, 0x0086, 0x0086, 0x0086, 0x0086, 0x0087,
+	0x0086, 0x0086, 0x0086, 0x0086, 0x0086, 0x0086, 0x0086, 0x0085,
+	0x0086, 0x0086, 0x0086, 0x0086, 0x0086, 0x0087, 0x0086, 0x0086,
+	0x0087, 0x0086, 0x0086, 0x0086, 0x0086, 0x0086, 0x0086, 0x0086,
+	0x0086, 0x0086, 0x0086, 0x0086, 0x0085, 0x0085, 0x0086, 0x0086,
+	0x0085, 0x0086, 0x0086, 0x0086, 0x0086, 0x0086, 0x0000, 0x00ee,
+	0x0000, 0x00f0, 0x00f1, 0x00f1, 0x00f1, 0x00f1, 0x00f1, 0x00f1,
+	0x00f1, 0x00f1, 0x00f1, 0x00f0, 0x00f1, 0x00f0, 0x00f0, 0x00f1,
+	// Entry 100 - 13F
+	0x00f1, 0x00f0, 0x00f1, 0x00f1, 0x00f1, 0x00f1, 0x00f0, 0x00f1,
+	0x00f1, 0x00f1, 0x00f1, 0x00f1, 0x00f1, 0x0000, 0x010d, 0x0000,
+	0x010f, 0x0000, 0x0111, 0x0000, 0x0113, 0x0113, 0x0000, 0x0116,
+	0x0116, 0x0116, 0x0116, 0x0000, 0x011b, 0x0000, 0x011d, 0x0000,
+	0x011f, 0x011f, 0x0000, 0x0122, 0x0122, 0x0122, 0x0122, 0x0122,
+	0x0122, 0x0122, 0x0122, 0x0122, 0x0122, 0x0122, 0x0122, 0x0122,
+	0x0122, 0x0122, 0x0122, 0x0122, 0x0122, 0x0122, 0x0122, 0x0122,
+	0x0122, 0x0122, 0x0122, 0x0122, 0x0122, 0x0122, 0x0122, 0x0122,
+	// Entry 140 - 17F
+	0x0122, 0x0122, 0x0122, 0x0122, 0x0122, 0x0122, 0x0122, 0x0122,
+	0x0122, 0x0122, 0x0122, 0x0122, 0x0122, 0x0122, 0x0122, 0x0122,
+	0x0122, 0x0000, 0x0151, 0x0000, 0x0153, 0x0000, 0x0155, 0x0000,
+	0x0157, 0x0000, 0x0159, 0x0000, 0x015b, 0x015b, 0x015b, 0x0000,
+	0x015f, 0x0000, 0x0000, 0x0162, 0x0000, 0x0164, 0x0000, 0x0166,
+	0x0166, 0x0166, 0x0000, 0x016a, 0x0000, 0x016c, 0x0000, 0x016e,
+	0x0000, 0x0170, 0x0170, 0x0000, 0x0173, 0x0000, 0x0175, 0x0000,
+	0x0177, 0x0000, 0x0179, 0x0000, 0x017b, 0x0000, 0x017d, 0x0000,
+	// Entry 180 - 1BF
+	0x017f, 0x0000, 0x0181, 0x0181, 0x0181, 0x0181, 0x0000, 0x0000,
+	0x0187, 0x0000, 0x0000, 0x018a, 0x0000, 0x018c, 0x0000, 0x0000,
+	0x018f, 0x0000, 0x0191, 0x0000, 0x0000, 0x0194, 0x0000, 0x0000,
+	0x0197, 0x0000, 0x0199, 0x0000, 0x019b, 0x0000, 0x019d, 0x0000,
+	0x019f, 0x0000, 0x01a1, 0x0000, 0x01a3, 0x0000, 0x01a5, 0x0000,
+	0x01a7, 0x0000, 0x01a9, 0x0000, 0x01ab, 0x01ab, 0x0000, 0x01ae,
+	0x0000, 0x01b0, 0x0000, 0x01b2, 0x0000, 0x01b4, 0x0000, 0x01b6,
+	0x0000, 0x0000, 0x01b9, 0x0000, 0x01bb, 0x0000, 0x01bd, 0x0000,
+	// Entry 1C0 - 1FF
+	0x01bf, 0x0000, 0x01c1, 0x0000, 0x01c3, 0x0000, 0x01c5, 0x01c5,
+	0x01c5, 0x01c5, 0x0000, 0x01ca, 0x0000, 0x01cc, 0x01cc, 0x0000,
+	0x01cf, 0x0000, 0x01d1, 0x0000, 0x01d3, 0x0000, 0x01d5, 0x0000,
+	0x01d7, 0x0000, 0x01d9, 0x01d9, 0x0000, 0x01dc, 0x0000, 0x01de,
+	0x0000, 0x01e0, 0x0000, 0x01e2, 0x0000, 0x01e4, 0x0000, 0x01e6,
+	0x0000, 0x01e8, 0x0000, 0x01ea, 0x0000, 0x01ec, 0x0000, 0x01ee,
+	0x01ee, 0x01ee, 0x0000, 0x01f2, 0x0000, 0x01f4, 0x0000, 0x01f6,
+	0x0000, 0x01f8, 0x0000, 0x0000, 0x01fb, 0x0000, 0x01fd, 0x01fd,
+	// Entry 200 - 23F
+	0x0000, 0x0200, 0x0000, 0x0202, 0x0202, 0x0000, 0x0205, 0x0205,
+	0x0000, 0x0208, 0x0208, 0x0208, 0x0208, 0x0208, 0x0208, 0x0208,
+	0x0000, 0x0210, 0x0000, 0x0212, 0x0000, 0x0214, 0x0000, 0x0000,
+	0x0000, 0x0000, 0x0000, 0x021a, 0x0000, 0x0000, 0x021d, 0x0000,
+	0x021f, 0x021f, 0x0000, 0x0222, 0x0000, 0x0224, 0x0224, 0x0000,
+	0x0000, 0x0228, 0x0227, 0x0227, 0x0000, 0x0000, 0x022d, 0x0000,
+	0x022f, 0x0000, 0x0231, 0x0000, 0x023d, 0x0233, 0x023d, 0x023d,
+	0x023d, 0x023d, 0x023d, 0x023d, 0x023d, 0x0233, 0x023d, 0x023d,
+	// Entry 240 - 27F
+	0x0000, 0x0240, 0x0240, 0x0240, 0x0000, 0x0244, 0x0000, 0x0246,
+	0x0000, 0x0248, 0x0248, 0x0000, 0x024b, 0x0000, 0x024d, 0x024d,
+	0x024d, 0x024d, 0x024d, 0x024d, 0x0000, 0x0254, 0x0000, 0x0256,
+	0x0000, 0x0258, 0x0000, 0x025a, 0x0000, 0x025c, 0x0000, 0x0000,
+	0x025f, 0x025f, 0x025f, 0x0000, 0x0263, 0x0000, 0x0265, 0x0000,
+	0x0267, 0x0000, 0x0000, 0x026a, 0x0269, 0x0269, 0x0000, 0x026e,
+	0x0000, 0x0270, 0x0000, 0x0272, 0x0000, 0x0000, 0x0000, 0x0000,
+	0x0277, 0x0000, 0x0000, 0x027a, 0x0000, 0x027c, 0x027c, 0x027c,
+	// Entry 280 - 2BF
+	0x027c, 0x0000, 0x0281, 0x0281, 0x0281, 0x0000, 0x0285, 0x0285,
+	0x0285, 0x0285, 0x0285, 0x0000, 0x028b, 0x028b, 0x028b, 0x028b,
+	0x0000, 0x0000, 0x0000, 0x0000, 0x0293, 0x0293, 0x0293, 0x0000,
+	0x0297, 0x0297, 0x0297, 0x0297, 0x0000, 0x0000, 0x029d, 0x029d,
+	0x029d, 0x029d, 0x0000, 0x02a2, 0x0000, 0x02a4, 0x02a4, 0x0000,
+	0x02a7, 0x0000, 0x02a9, 0x02a9, 0x0000, 0x0000, 0x02ad, 0x0000,
+	0x0000, 0x02b0, 0x0000, 0x02b2, 0x02b2, 0x0000, 0x0000, 0x02b6,
+	0x0000, 0x02b8, 0x0000, 0x02ba, 0x0000, 0x02bc, 0x0000, 0x02be,
+	// Entry 2C0 - 2FF
+	0x02be, 0x0000, 0x0000, 0x02c2, 0x0000, 0x02c4, 0x02c1, 0x02c1,
+	0x0000, 0x0000, 0x02c9, 0x02c8, 0x02c8, 0x0000, 0x0000, 0x02ce,
+	0x0000, 0x02d0, 0x0000, 0x02d2, 0x0000, 0x0000, 0x02d5, 0x0000,
+	0x0000, 0x0000, 0x02d9, 0x0000, 0x02db, 0x0000, 0x02dd, 0x0000,
+	0x02df, 0x02df, 0x0000, 0x02e2, 0x0000, 0x02e4, 0x0000, 0x02e6,
+	0x02e6, 0x02e6, 0x02e6, 0x02e6, 0x0000, 0x02ec, 0x02ed, 0x02ec,
+	0x0000, 0x02f0,
+} // Size: 1532 bytes
+
+// Total table size 1532 bytes (1KiB); checksum: 90718A2

vendor/golang.org/x/text/language/common.go

@@ -1,4 +1,4 @@
-// This file was generated by go generate; DO NOT EDIT
+// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
 
 package language
 

vendor/golang.org/x/text/language/gen.go

@@ -678,6 +678,8 @@ func (b *builder) parseIndices() {
 	b.locale.parse(meta.DefaultContent.Locales)
 }
 
+// TODO: region inclusion data will probably not be use used in future matchers.
+
 func (b *builder) computeRegionGroups() {
 	b.groups = make(map[int]index)
 
@@ -686,6 +688,11 @@ func (b *builder) computeRegionGroups() {
 		b.groups[i] = index(len(b.groups))
 	}
 	for _, g := range b.supp.TerritoryContainment.Group {
+		// Skip UN and EURO zone as they are flattening the containment
+		// relationship.
+		if g.Type == "EZ" || g.Type == "UN" {
+			continue
+		}
 		group := b.region.index(g.Type)
 		if _, ok := b.groups[group]; !ok {
 			b.groups[group] = index(len(b.groups))
@@ -782,6 +789,7 @@ func (b *builder) writeLanguage() {
 	lang.updateLater("tw", "twi")
 	lang.updateLater("nb", "nob")
 	lang.updateLater("ak", "aka")
+	lang.updateLater("bh", "bih")
 
 	// Ensure that each 2-letter code is matched with a 3-letter code.
 	for _, v := range lang.s[1:] {
@@ -798,10 +806,10 @@ func (b *builder) writeLanguage() {
 		}
 	}
 
-	// Complete canonialized language tags.
+	// Complete canonicalized language tags.
 	lang.freeze()
 	for i, v := range lang.s {
-		// We can avoid these manual entries by using the IANI registry directly.
+		// We can avoid these manual entries by using the IANA registry directly.
 		// Seems easier to update the list manually, as changes are rare.
 		// The panic in this loop will trigger if we miss an entry.
 		add := ""
@@ -908,7 +916,7 @@ func (b *builder) writeRegion() {
 			i := b.region.index(s)
 			for _, d := range e.description {
 				if strings.Contains(d, "Private use") {
-					regionTypes[i] = iso3166UserAssgined
+					regionTypes[i] = iso3166UserAssigned
 				}
 			}
 			regionTypes[i] |= bcp47Region
@@ -1065,7 +1073,7 @@ const (
 )
 
 const (
-	iso3166UserAssgined = 1 << iota
+	iso3166UserAssigned = 1 << iota
 	ccTLD
 	bcp47Region
 )
@@ -1355,42 +1363,23 @@ func (b *builder) writeLikelyData() {
 
 type mutualIntelligibility struct {
 	want, have uint16
-	conf       uint8
+	distance   uint8
 	oneway     bool
 }
 
 type scriptIntelligibility struct {
-	lang       uint16 // langID or 0 if *
-	want, have uint8
-	conf       uint8
+	wantLang, haveLang     uint16
+	wantScript, haveScript uint8
+	distance               uint8
+	// Always oneway
 }
 
-type sortByConf []mutualIntelligibility
-
-func (l sortByConf) Less(a, b int) bool {
-	return l[a].conf > l[b].conf
-}
-
-func (l sortByConf) Swap(a, b int) {
-	l[a], l[b] = l[b], l[a]
-}
-
-func (l sortByConf) Len() int {
-	return len(l)
-}
-
-// toConf converts a percentage value [0, 100] to a confidence class.
-func toConf(pct uint8) uint8 {
-	switch {
-	case pct == 100:
-		return 3 // Exact
-	case pct >= 90:
-		return 2 // High
-	case pct > 50:
-		return 1 // Low
-	default:
-		return 0 // No
-	}
+type regionIntelligibility struct {
+	lang     uint16 // compact language id
+	script   uint8  // 0 means any
+	group    uint8  // 0 means any; if bit 7 is set it means inverse
+	distance uint8
+	// Always twoway.
 }
 
 // writeMatchData writes tables with languages and scripts for which there is
@@ -1400,13 +1389,50 @@ func toConf(pct uint8) uint8 {
 // We also drop all region-related data as we use a different algorithm to
 // determine region equivalence.
 func (b *builder) writeMatchData() {
+	lm := b.supp.LanguageMatching.LanguageMatches
+	cldr.MakeSlice(&lm).SelectAnyOf("type", "written_new")
+
+	regionHierarchy := map[string][]string{}
+	for _, g := range b.supp.TerritoryContainment.Group {
+		regions := strings.Split(g.Contains, " ")
+		regionHierarchy[g.Type] = append(regionHierarchy[g.Type], regions...)
+	}
+	regionToGroups := make([]uint8, len(b.region.s))
+
+	idToIndex := map[string]uint8{}
+	for i, mv := range lm[0].MatchVariable {
+		if i > 6 {
+			log.Fatalf("Too many groups: %d", i)
+		}
+		idToIndex[mv.Id] = uint8(i + 1)
+		// TODO: also handle '-'
+		for _, r := range strings.Split(mv.Value, "+") {
+			todo := []string{r}
+			for k := 0; k < len(todo); k++ {
+				r := todo[k]
+				regionToGroups[b.region.index(r)] |= 1 << uint8(i)
+				todo = append(todo, regionHierarchy[r]...)
+			}
+		}
+	}
+	b.writeSlice("regionToGroups", regionToGroups)
+
 	b.writeType(mutualIntelligibility{})
 	b.writeType(scriptIntelligibility{})
-	lm := b.supp.LanguageMatching.LanguageMatches
-	cldr.MakeSlice(&lm).SelectAnyOf("type", "written")
+	b.writeType(regionIntelligibility{})
 
-	matchLang := []mutualIntelligibility{}
+	matchLang := []mutualIntelligibility{{
+		// TODO: remove once CLDR is fixed.
+		want:     uint16(b.langIndex("sr")),
+		have:     uint16(b.langIndex("hr")),
+		distance: uint8(5),
+	}, {
+		want:     uint16(b.langIndex("sr")),
+		have:     uint16(b.langIndex("bs")),
+		distance: uint8(5),
+	}}
 	matchScript := []scriptIntelligibility{}
+	matchRegion := []regionIntelligibility{}
 	// Convert the languageMatch entries in lists keyed by desired language.
 	for _, m := range lm[0].LanguageMatch {
 		// Different versions of CLDR use different separators.
@@ -1414,33 +1440,38 @@ func (b *builder) writeMatchData() {
 		supported := strings.Replace(m.Supported, "-", "_", -1)
 		d := strings.Split(desired, "_")
 		s := strings.Split(supported, "_")
-		if len(d) != len(s) || len(d) > 2 {
-			// Skip all entries with regions and work around CLDR bug.
+		if len(d) != len(s) {
+			log.Fatalf("not supported: desired=%q; supported=%q", desired, supported)
 			continue
 		}
-		pct, _ := strconv.ParseInt(m.Percent, 10, 8)
-		if len(d) == 2 && d[0] == s[0] && len(d[1]) == 4 {
-			// language-script pair.
-			lang := uint16(0)
-			if d[0] != "*" {
-				lang = uint16(b.langIndex(d[0]))
+		distance, _ := strconv.ParseInt(m.Distance, 10, 8)
+		switch len(d) {
+		case 2:
+			if desired == supported && desired == "*_*" {
+				continue
 			}
+			// language-script pair.
 			matchScript = append(matchScript, scriptIntelligibility{
-				lang: lang,
-				want: uint8(b.script.index(d[1])),
-				have: uint8(b.script.index(s[1])),
-				conf: toConf(uint8(pct)),
+				wantLang:   uint16(b.langIndex(d[0])),
+				haveLang:   uint16(b.langIndex(s[0])),
+				wantScript: uint8(b.script.index(d[1])),
+				haveScript: uint8(b.script.index(s[1])),
+				distance:   uint8(distance),
 			})
 			if m.Oneway != "true" {
 				matchScript = append(matchScript, scriptIntelligibility{
-					lang: lang,
-					want: uint8(b.script.index(s[1])),
-					have: uint8(b.script.index(d[1])),
-					conf: toConf(uint8(pct)),
+					wantLang:   uint16(b.langIndex(s[0])),
+					haveLang:   uint16(b.langIndex(d[0])),
+					wantScript: uint8(b.script.index(s[1])),
+					haveScript: uint8(b.script.index(d[1])),
+					distance:   uint8(distance),
 				})
 			}
-		} else if len(d) == 1 && d[0] != "*" {
-			if pct == 100 {
+		case 1:
+			if desired == supported && desired == "*" {
+				continue
+			}
+			if distance == 1 {
 				// nb == no is already handled by macro mapping. Check there
 				// really is only this case.
 				if d[0] != "no" || s[0] != "nb" {
@@ -1448,28 +1479,57 @@ func (b *builder) writeMatchData() {
 				}
 				continue
 			}
+			// TODO: consider dropping oneway field and just doubling the entry.
 			matchLang = append(matchLang, mutualIntelligibility{
-				want:   uint16(b.langIndex(d[0])),
-				have:   uint16(b.langIndex(s[0])),
-				conf:   uint8(pct),
-				oneway: m.Oneway == "true",
+				want:     uint16(b.langIndex(d[0])),
+				have:     uint16(b.langIndex(s[0])),
+				distance: uint8(distance),
+				oneway:   m.Oneway == "true",
 			})
-		} else {
-			// TODO: Handle other mappings.
-			a := []string{"*;*", "*_*;*_*", "es_MX;es_419"}
-			s := strings.Join([]string{desired, supported}, ";")
-			if i := sort.SearchStrings(a, s); i == len(a) || a[i] != s {
-				log.Printf("%q not handled", s)
+		case 3:
+			if desired == supported && desired == "*_*_*" {
+				continue
 			}
+			if desired != supported { // (Weird but correct.)
+				log.Fatalf("not supported: desired=%q; supported=%q", desired, supported)
+				continue
+			}
+			ri := regionIntelligibility{
+				lang:     b.langIndex(d[0]),
+				distance: uint8(distance),
+			}
+			if d[1] != "*" {
+				ri.script = uint8(b.script.index(d[1]))
+			}
+			switch {
+			case d[2] == "*":
+				ri.group = 0x80 // not contained in anything
+			case strings.HasPrefix(d[2], "$!"):
+				ri.group = 0x80
+				d[2] = "$" + d[2][len("$!"):]
+				fallthrough
+			case strings.HasPrefix(d[2], "$"):
+				ri.group |= idToIndex[d[2]]
+			}
+			matchRegion = append(matchRegion, ri)
+		default:
+			log.Fatalf("not supported: desired=%q; supported=%q", desired, supported)
 		}
 	}
-	sort.Stable(sortByConf(matchLang))
-	// collapse percentage into confidence classes
-	for i, m := range matchLang {
-		matchLang[i].conf = toConf(m.conf)
-	}
+	sort.SliceStable(matchLang, func(i, j int) bool {
+		return matchLang[i].distance < matchLang[j].distance
+	})
 	b.writeSlice("matchLang", matchLang)
+
+	sort.SliceStable(matchScript, func(i, j int) bool {
+		return matchScript[i].distance < matchScript[j].distance
+	})
 	b.writeSlice("matchScript", matchScript)
+
+	sort.SliceStable(matchRegion, func(i, j int) bool {
+		return matchRegion[i].distance < matchRegion[j].distance
+	})
+	b.writeSlice("matchRegion", matchRegion)
 }
 
 func (b *builder) writeRegionInclusionData() {
@@ -1482,6 +1542,11 @@ func (b *builder) writeRegionInclusionData() {
 		containment = make(map[index][]index)
 	)
 	for _, g := range b.supp.TerritoryContainment.Group {
+		// Skip UN and EURO zone as they are flattening the containment
+		// relationship.
+		if g.Type == "EZ" || g.Type == "UN" {
+			continue
+		}
 		group := b.region.index(g.Type)
 		groupIdx := b.groups[group]
 		for _, mem := range strings.Split(g.Contains, " ") {
@@ -1508,7 +1573,6 @@ func (b *builder) writeRegionInclusionData() {
 		for _, v := range l {
 			regionContainment[g] |= 1 << v
 		}
-		// log.Printf("%d: %X", g, regionContainment[g])
 	}
 	b.writeSlice("regionContainment", regionContainment)
 

vendor/golang.org/x/text/language/language.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-//go:generate go run maketables.go gen_common.go -output tables.go
+//go:generate go run gen.go gen_common.go -output tables.go
 //go:generate go run gen_index.go
 
 // Package language implements BCP 47 language tags and related functionality.
@@ -129,8 +129,15 @@ const (
 // specific language or locale. All language tag values are guaranteed to be
 // well-formed.
 type Tag struct {
-	lang     langID
-	region   regionID
+	lang   langID
+	region regionID
+	// TODO: we will soon run out of positions for script. Idea: instead of
+	// storing lang, region, and script codes, store only the compact index and
+	// have a lookup table from this code to its expansion. This greatly speeds
+	// up table lookup, speed up common variant cases.
+	// This will also immediately free up 3 extra bytes. Also, the pVariant
+	// field can now be moved to the lookup table, as the compact index uniquely
+	// determines the offset of a possible variant.
 	script   scriptID
 	pVariant byte   // offset in str, includes preceding '-'
 	pExt     uint16 // offset of first extension, includes preceding '-'
@@ -593,7 +600,7 @@ func (t Tag) Extension(x byte) (ext Extension, ok bool) {
 			return Extension{ext}, true
 		}
 	}
-	return Extension{string(x)}, false
+	return Extension{}, false
 }
 
 // Extensions returns all extensions of t.

vendor/golang.org/x/text/language/match.go

@@ -6,6 +6,16 @@ package language
 
 import "errors"
 
+// A MatchOption configures a Matcher.
+type MatchOption func(*matcher)
+
+// PreferSameScript will, in the absence of a match, result in the first
+// preferred tag with the same script as a supported tag to match this supported
+// tag. The default is currently true, but this may change in the future.
+func PreferSameScript(preferSame bool) MatchOption {
+	return func(m *matcher) { m.preferSameScript = preferSame }
+}
+
 // Matcher is the interface that wraps the Match method.
 //
 // Match returns the best match for any of the given tags, along with
@@ -36,23 +46,44 @@ func Comprehends(speaker, alternative Tag) Confidence {
 // matched tag in t, but is augmented with the Unicode extension ('u')of the
 // corresponding preferred tag. This allows user locale options to be passed
 // transparently.
-func NewMatcher(t []Tag) Matcher {
-	return newMatcher(t)
+func NewMatcher(t []Tag, options ...MatchOption) Matcher {
+	return newMatcher(t, options)
 }
 
 func (m *matcher) Match(want ...Tag) (t Tag, index int, c Confidence) {
 	match, w, c := m.getBest(want...)
-	if match == nil {
-		t = m.default_.tag
-	} else {
+	if match != nil {
 		t, index = match.tag, match.index
+	} else {
+		// TODO: this should be an option
+		t = m.default_.tag
+		if m.preferSameScript {
+		outer:
+			for _, w := range want {
+				script, _ := w.Script()
+				if script.scriptID == 0 {
+					// Don't do anything if there is no script, such as with
+					// private subtags.
+					continue
+				}
+				for i, h := range m.supported {
+					if script.scriptID == h.maxScript {
+						t, index = h.tag, i
+						break outer
+					}
+				}
+			}
+		}
+		// TODO: select first language tag based on script.
+	}
+	if w.region != 0 && t.region != 0 && t.region.contains(w.region) {
+		t, _ = Raw.Compose(t, Region{w.region})
 	}
 	// Copy options from the user-provided tag into the result tag. This is hard
 	// to do after the fact, so we do it here.
-	// TODO: consider also adding in variants that are compatible with the
-	// matched language.
-	// TODO: Add back region if it is non-ambiguous? Or create another tag to
-	// preserve the region?
+	// TODO: add in alternative variants to -u-va-.
+	// TODO: add preferred region to -u-rg-.
+	// TODO: add other extensions. Merge with existing extensions.
 	if u, ok := w.Extension('u'); ok {
 		t, _ = Raw.Compose(t, u)
 	}
@@ -388,16 +419,18 @@ func minimizeTags(t Tag) (Tag, error) {
 
 // matcher keeps a set of supported language tags, indexed by language.
 type matcher struct {
-	default_     *haveTag
-	index        map[langID]*matchHeader
-	passSettings bool
+	default_         *haveTag
+	supported        []*haveTag
+	index            map[langID]*matchHeader
+	passSettings     bool
+	preferSameScript bool
 }
 
 // matchHeader has the lists of tags for exact matches and matches based on
 // maximized and canonicalized tags for a given language.
 type matchHeader struct {
-	exact []haveTag
-	max   []haveTag
+	exact []*haveTag
+	max   []*haveTag
 }
 
 // haveTag holds a supported Tag and its maximized script and region. The maximized
@@ -440,8 +473,10 @@ func makeHaveTag(tag Tag, index int) (haveTag, langID) {
 // script to map to another and we rely on this to keep the code simple.
 func altScript(l langID, s scriptID) scriptID {
 	for _, alt := range matchScript {
-		if (alt.lang == 0 || langID(alt.lang) == l) && scriptID(alt.have) == s {
-			return scriptID(alt.want)
+		// TODO: also match cases where language is not the same.
+		if (langID(alt.wantLang) == l || langID(alt.haveLang) == l) &&
+			scriptID(alt.haveScript) == s {
+			return scriptID(alt.wantScript)
 		}
 	}
 	return 0
@@ -457,7 +492,7 @@ func (h *matchHeader) addIfNew(n haveTag, exact bool) {
 		}
 	}
 	if exact {
-		h.exact = append(h.exact, n)
+		h.exact = append(h.exact, &n)
 	}
 	// Allow duplicate maximized tags, but create a linked list to allow quickly
 	// comparing the equivalents and bail out.
@@ -472,7 +507,7 @@ func (h *matchHeader) addIfNew(n haveTag, exact bool) {
 			break
 		}
 	}
-	h.max = append(h.max, n)
+	h.max = append(h.max, &n)
 }
 
 // header returns the matchHeader for the given language. It creates one if
@@ -486,12 +521,26 @@ func (m *matcher) header(l langID) *matchHeader {
 	return h
 }
 
+func toConf(d uint8) Confidence {
+	if d <= 10 {
+		return High
+	}
+	if d < 30 {
+		return Low
+	}
+	return No
+}
+
 // newMatcher builds an index for the given supported tags and returns it as
 // a matcher. It also expands the index by considering various equivalence classes
 // for a given tag.
-func newMatcher(supported []Tag) *matcher {
+func newMatcher(supported []Tag, options []MatchOption) *matcher {
 	m := &matcher{
-		index: make(map[langID]*matchHeader),
+		index:            make(map[langID]*matchHeader),
+		preferSameScript: true,
+	}
+	for _, o := range options {
+		o(m)
 	}
 	if len(supported) == 0 {
 		m.default_ = &haveTag{}
@@ -502,8 +551,9 @@ func newMatcher(supported []Tag) *matcher {
 	for i, tag := range supported {
 		pair, _ := makeHaveTag(tag, i)
 		m.header(tag.lang).addIfNew(pair, true)
+		m.supported = append(m.supported, &pair)
 	}
-	m.default_ = &m.header(supported[0].lang).exact[0]
+	m.default_ = m.header(supported[0].lang).exact[0]
 	for i, tag := range supported {
 		pair, max := makeHaveTag(tag, i)
 		if max != tag.lang {
@@ -511,6 +561,9 @@ func newMatcher(supported []Tag) *matcher {
 		}
 	}
 
+	// TODO: include alt script.
+	// - don't replace regions, but allow regions to be made more specific.
+
 	// update is used to add indexes in the map for equivalent languages.
 	// If force is true, the update will also apply to derived entries. To
 	// avoid applying a "transitive closure", use false.
@@ -520,7 +573,8 @@ func newMatcher(supported []Tag) *matcher {
 				return
 			}
 			hw := m.header(langID(want))
-			for _, v := range hh.max {
+			for _, ht := range hh.max {
+				v := *ht
 				if conf < v.conf {
 					v.conf = conf
 				}
@@ -536,9 +590,9 @@ func newMatcher(supported []Tag) *matcher {
 	// Add entries for languages with mutual intelligibility as defined by CLDR's
 	// languageMatch data.
 	for _, ml := range matchLang {
-		update(ml.want, ml.have, Confidence(ml.conf), false)
+		update(ml.want, ml.have, toConf(ml.distance), false)
 		if !ml.oneway {
-			update(ml.have, ml.want, Confidence(ml.conf), false)
+			update(ml.have, ml.want, toConf(ml.distance), false)
 		}
 	}
 
@@ -580,7 +634,7 @@ func (m *matcher) getBest(want ...Tag) (got *haveTag, orig Tag, c Confidence) {
 				continue
 			}
 			for i := range h.exact {
-				have := &h.exact[i]
+				have := h.exact[i]
 				if have.tag.equalsRest(w) {
 					return have, w, Exact
 				}
@@ -591,7 +645,7 @@ func (m *matcher) getBest(want ...Tag) (got *haveTag, orig Tag, c Confidence) {
 			// Base language is not defined.
 			if h != nil {
 				for i := range h.exact {
-					have := &h.exact[i]
+					have := h.exact[i]
 					if have.tag.equalsRest(w) {
 						return have, w, Exact
 					}
@@ -609,11 +663,11 @@ func (m *matcher) getBest(want ...Tag) (got *haveTag, orig Tag, c Confidence) {
 		}
 		// Check for match based on maximized tag.
 		for i := range h.max {
-			have := &h.max[i]
+			have := h.max[i]
 			best.update(have, w, max.script, max.region)
 			if best.conf == Exact {
 				for have.nextMax != 0 {
-					have = &h.max[have.nextMax]
+					have = h.max[have.nextMax]
 					best.update(have, w, max.script, max.region)
 				}
 				return best.have, best.want, High
@@ -635,11 +689,12 @@ type bestMatch struct {
 	want Tag
 	conf Confidence
 	// Cached results from applying tie-breaking rules.
-	origLang   bool
-	origReg    bool
-	regDist    uint8
-	origScript bool
-	parentDist uint8 // 255 if have is not an ancestor of want tag.
+	origLang     bool
+	origReg      bool
+	regGroupDist uint8
+	regDist      uint8
+	origScript   bool
+	parentDist   uint8 // 255 if have is not an ancestor of want tag.
 }
 
 // update updates the existing best match if the new pair is considered to be a
@@ -693,6 +748,14 @@ func (m *bestMatch) update(have *haveTag, tag Tag, maxScript scriptID, maxRegion
 		beaten = true
 	}
 
+	regGroupDist := regionGroupDist(have.maxRegion, maxRegion, maxScript, tag.lang)
+	if !beaten && m.regGroupDist != regGroupDist {
+		if regGroupDist > m.regGroupDist {
+			return
+		}
+		beaten = true
+	}
+
 	// We prefer if the pre-maximized region was specified and identical.
 	origReg := have.tag.region == tag.region && tag.region != 0
 	if !beaten && m.origReg != origReg {
@@ -702,8 +765,22 @@ func (m *bestMatch) update(have *haveTag, tag Tag, maxScript scriptID, maxRegion
 		beaten = true
 	}
 
-	// Next we prefer smaller distances between regions, as defined by regionDist.
-	regDist := regionDist(have.maxRegion, maxRegion, tag.lang)
+	// TODO: remove the region distance rule. Region distance has been replaced
+	// by the region grouping rule. For now we leave it as it still seems to
+	// have a net positive effect when applied after the grouping rule.
+	// Possible solutions:
+	// - apply the primary locale rule first to effectively disable region
+	//   region distance if groups are defined.
+	// - express the following errors in terms of grouping (if possible)
+	// - find another method of handling the following cases.
+	// maximization of legacy: find mo in
+	//      "sr-Cyrl, sr-Latn, ro, ro-MD": have ro; want ro-MD (High)
+	// region distance French: find fr-US in
+	//      "en, fr, fr-CA, fr-CH": have fr; want fr-CA (High)
+
+	// Next we prefer smaller distances between regions, as defined by
+	// regionDist.
+	regDist := uint8(regionDistance(have.maxRegion, maxRegion))
 	if !beaten && m.regDist != regDist {
 		if regDist > m.regDist {
 			return
@@ -721,6 +798,9 @@ func (m *bestMatch) update(have *haveTag, tag Tag, maxScript scriptID, maxRegion
 	}
 
 	// Finally we prefer tags which have a closer parent relationship.
+	// TODO: the parent relationship no longer seems necessary. It doesn't hurt
+	// to leave it in as the final tie-breaker, though, especially until the
+	// grouping data has further matured.
 	parentDist := parentDistance(have.tag.region, tag)
 	if !beaten && m.parentDist != parentDist {
 		if parentDist > m.parentDist {
@@ -737,6 +817,7 @@ func (m *bestMatch) update(have *haveTag, tag Tag, maxScript scriptID, maxRegion
 		m.origLang = origLang
 		m.origReg = origReg
 		m.origScript = origScript
+		m.regGroupDist = regGroupDist
 		m.regDist = regDist
 		m.parentDist = parentDist
 	}
@@ -759,15 +840,27 @@ func parentDistance(haveRegion regionID, tag Tag) uint8 {
 	return d
 }
 
-// regionDist wraps regionDistance with some exceptions to the algorithmic distance.
-func regionDist(a, b regionID, lang langID) uint8 {
-	if lang == _en {
-		// Two variants of non-US English are close to each other, regardless of distance.
-		if a != _US && b != _US {
-			return 2
+// regionGroupDist computes the distance between two regions based on their
+// CLDR grouping.
+func regionGroupDist(a, b regionID, script scriptID, lang langID) uint8 {
+	aGroup := uint(regionToGroups[a]) << 1
+	bGroup := uint(regionToGroups[b]) << 1
+	for _, ri := range matchRegion {
+		if langID(ri.lang) == lang && (ri.script == 0 || scriptID(ri.script) == script) {
+			group := uint(1 << (ri.group &^ 0x80))
+			if 0x80&ri.group == 0 {
+				if aGroup&bGroup&group != 0 { // Both regions are in the group.
+					return ri.distance
+				}
+			} else {
+				if (aGroup|bGroup)&group == 0 { // Both regions are not in the group.
+					return ri.distance
+				}
+			}
 		}
 	}
-	return uint8(regionDistance(a, b))
+	const defaultDistance = 4
+	return defaultDistance
 }
 
 // regionDistance computes the distance between two regions based on the

vendor/golang.org/x/text/runes/cond.go

@@ -41,20 +41,35 @@ func If(s Set, tIn, tNotIn transform.Transformer) Transformer {
 	if tNotIn == nil {
 		tNotIn = transform.Nop
 	}
+	sIn, ok := tIn.(transform.SpanningTransformer)
+	if !ok {
+		sIn = dummySpan{tIn}
+	}
+	sNotIn, ok := tNotIn.(transform.SpanningTransformer)
+	if !ok {
+		sNotIn = dummySpan{tNotIn}
+	}
+
 	a := &cond{
-		tIn:    tIn,
-		tNotIn: tNotIn,
+		tIn:    sIn,
+		tNotIn: sNotIn,
 		f:      s.Contains,
 	}
 	a.Reset()
 	return Transformer{a}
 }
 
+type dummySpan struct{ transform.Transformer }
+
+func (d dummySpan) Span(src []byte, atEOF bool) (n int, err error) {
+	return 0, transform.ErrEndOfSpan
+}
+
 type cond struct {
-	tIn, tNotIn transform.Transformer
+	tIn, tNotIn transform.SpanningTransformer
 	f           func(rune) bool
-	check       func(rune) bool       // current check to perform
-	t           transform.Transformer // current transformer to use
+	check       func(rune) bool               // current check to perform
+	t           transform.SpanningTransformer // current transformer to use
 }
 
 // Reset implements transform.Transformer.
@@ -84,6 +99,51 @@ func (t *cond) isNot(r rune) bool {
 	return false
 }
 
+// This implementation of Span doesn't help all too much, but it needs to be
+// there to satisfy this package's Transformer interface.
+// TODO: there are certainly room for improvements, though. For example, if
+// t.t == transform.Nop (which will a common occurrence) it will save a bundle
+// to special-case that loop.
+func (t *cond) Span(src []byte, atEOF bool) (n int, err error) {
+	p := 0
+	for n < len(src) && err == nil {
+		// Don't process too much at a time as the Spanner that will be
+		// called on this block may terminate early.
+		const maxChunk = 4096
+		max := len(src)
+		if v := n + maxChunk; v < max {
+			max = v
+		}
+		atEnd := false
+		size := 0
+		current := t.t
+		for ; p < max; p += size {
+			r := rune(src[p])
+			if r < utf8.RuneSelf {
+				size = 1
+			} else if r, size = utf8.DecodeRune(src[p:]); size == 1 {
+				if !atEOF && !utf8.FullRune(src[p:]) {
+					err = transform.ErrShortSrc
+					break
+				}
+			}
+			if !t.check(r) {
+				// The next rune will be the start of a new run.
+				atEnd = true
+				break
+			}
+		}
+		n2, err2 := current.Span(src[n:p], atEnd || (atEOF && p == len(src)))
+		n += n2
+		if err2 != nil {
+			return n, err2
+		}
+		// At this point either err != nil or t.check will pass for the rune at p.
+		p = n + size
+	}
+	return n, err
+}
+
 func (t *cond) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
 	p := 0
 	for nSrc < len(src) && err == nil {
@@ -99,9 +159,10 @@ func (t *cond) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error
 		size := 0
 		current := t.t
 		for ; p < max; p += size {
-			var r rune
-			r, size = utf8.DecodeRune(src[p:])
-			if r == utf8.RuneError && size == 1 {
+			r := rune(src[p])
+			if r < utf8.RuneSelf {
+				size = 1
+			} else if r, size = utf8.DecodeRune(src[p:]); size == 1 {
 				if !atEOF && !utf8.FullRune(src[p:]) {
 					err = transform.ErrShortSrc
 					break

vendor/golang.org/x/text/runes/runes.go

@@ -46,9 +46,19 @@ func Predicate(f func(rune) bool) Set {
 
 // Transformer implements the transform.Transformer interface.
 type Transformer struct {
-	transform.Transformer
+	t transform.SpanningTransformer
 }
 
+func (t Transformer) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
+	return t.t.Transform(dst, src, atEOF)
+}
+
+func (t Transformer) Span(b []byte, atEOF bool) (n int, err error) {
+	return t.t.Span(b, atEOF)
+}
+
+func (t Transformer) Reset() { t.t.Reset() }
+
 // Bytes returns a new byte slice with the result of converting b using t.  It
 // calls Reset on t. It returns nil if any error was found. This can only happen
 // if an error-producing Transformer is passed to If.
@@ -96,39 +106,57 @@ type remove func(r rune) bool
 
 func (remove) Reset() {}
 
+// Span implements transform.Spanner.
+func (t remove) Span(src []byte, atEOF bool) (n int, err error) {
+	for r, size := rune(0), 0; n < len(src); {
+		if r = rune(src[n]); r < utf8.RuneSelf {
+			size = 1
+		} else if r, size = utf8.DecodeRune(src[n:]); size == 1 {
+			// Invalid rune.
+			if !atEOF && !utf8.FullRune(src[n:]) {
+				err = transform.ErrShortSrc
+			} else {
+				err = transform.ErrEndOfSpan
+			}
+			break
+		}
+		if t(r) {
+			err = transform.ErrEndOfSpan
+			break
+		}
+		n += size
+	}
+	return
+}
+
 // Transform implements transform.Transformer.
 func (t remove) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
 	for r, size := rune(0), 0; nSrc < len(src); {
 		if r = rune(src[nSrc]); r < utf8.RuneSelf {
 			size = 1
-		} else {
-			r, size = utf8.DecodeRune(src[nSrc:])
-
-			if size == 1 {
-				// Invalid rune.
-				if !atEOF && !utf8.FullRune(src[nSrc:]) {
-					err = transform.ErrShortSrc
+		} else if r, size = utf8.DecodeRune(src[nSrc:]); size == 1 {
+			// Invalid rune.
+			if !atEOF && !utf8.FullRune(src[nSrc:]) {
+				err = transform.ErrShortSrc
+				break
+			}
+			// We replace illegal bytes with RuneError. Not doing so might
+			// otherwise turn a sequence of invalid UTF-8 into valid UTF-8.
+			// The resulting byte sequence may subsequently contain runes
+			// for which t(r) is true that were passed unnoticed.
+			if !t(utf8.RuneError) {
+				if nDst+3 > len(dst) {
+					err = transform.ErrShortDst
 					break
 				}
-				// We replace illegal bytes with RuneError. Not doing so might
-				// otherwise turn a sequence of invalid UTF-8 into valid UTF-8.
-				// The resulting byte sequence may subsequently contain runes
-				// for which t(r) is true that were passed unnoticed.
-				if !t(utf8.RuneError) {
-					if nDst+3 > len(dst) {
-						err = transform.ErrShortDst
-						break
-					}
-					dst[nDst+0] = runeErrorString[0]
-					dst[nDst+1] = runeErrorString[1]
-					dst[nDst+2] = runeErrorString[2]
-					nDst += 3
-				}
-				nSrc++
-				continue
+				dst[nDst+0] = runeErrorString[0]
+				dst[nDst+1] = runeErrorString[1]
+				dst[nDst+2] = runeErrorString[2]
+				nDst += 3
 			}
+			nSrc++
+			continue
 		}
-
 		if t(r) {
 			nSrc += size
 			continue
@@ -157,6 +185,28 @@ type mapper func(rune) rune
 
 func (mapper) Reset() {}
 
+// Span implements transform.Spanner.
+func (t mapper) Span(src []byte, atEOF bool) (n int, err error) {
+	for r, size := rune(0), 0; n < len(src); n += size {
+		if r = rune(src[n]); r < utf8.RuneSelf {
+			size = 1
+		} else if r, size = utf8.DecodeRune(src[n:]); size == 1 {
+			// Invalid rune.
+			if !atEOF && !utf8.FullRune(src[n:]) {
+				err = transform.ErrShortSrc
+			} else {
+				err = transform.ErrEndOfSpan
+			}
+			break
+		}
+		if t(r) != r {
+			err = transform.ErrEndOfSpan
+			break
+		}
+	}
+	return n, err
+}
+
 // Transform implements transform.Transformer.
 func (t mapper) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
 	var replacement rune
@@ -230,24 +280,51 @@ func ReplaceIllFormed() Transformer {
 
 type replaceIllFormed struct{ transform.NopResetter }
 
+func (t replaceIllFormed) Span(src []byte, atEOF bool) (n int, err error) {
+	for n < len(src) {
+		// ASCII fast path.
+		if src[n] < utf8.RuneSelf {
+			n++
+			continue
+		}
+
+		r, size := utf8.DecodeRune(src[n:])
+
+		// Look for a valid non-ASCII rune.
+		if r != utf8.RuneError || size != 1 {
+			n += size
+			continue
+		}
+
+		// Look for short source data.
+		if !atEOF && !utf8.FullRune(src[n:]) {
+			err = transform.ErrShortSrc
+			break
+		}
+
+		// We have an invalid rune.
+		err = transform.ErrEndOfSpan
+		break
+	}
+	return n, err
+}
+
 func (t replaceIllFormed) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
 	for nSrc < len(src) {
-		r, size := utf8.DecodeRune(src[nSrc:])
-
-		// Look for an ASCII rune.
-		if r < utf8.RuneSelf {
+		// ASCII fast path.
+		if r := src[nSrc]; r < utf8.RuneSelf {
 			if nDst == len(dst) {
 				err = transform.ErrShortDst
 				break
 			}
-			dst[nDst] = byte(r)
+			dst[nDst] = r
 			nDst++
 			nSrc++
 			continue
 		}
 
 		// Look for a valid non-ASCII rune.
-		if r != utf8.RuneError || size != 1 {
+		if _, size := utf8.DecodeRune(src[nSrc:]); size != 1 {
 			if size != copy(dst[nDst:], src[nSrc:nSrc+size]) {
 				err = transform.ErrShortDst
 				break

vendor/golang.org/x/text/secure/bidirule/bidirule.go

@@ -123,34 +123,64 @@ var transitions = [...][2]ruleTransition{
 // vice versa.
 const exclusiveRTL = uint16(1<<bidi.EN | 1<<bidi.AN)
 
-// Direction reports the direction of the given label as defined by RFC 5893 or
-// an error if b is not a valid label according to the Bidi Rule.
-func Direction(b []byte) (bidi.Direction, error) {
-	t := Transformer{}
-	if n, ok := t.advance(b); ok && n == len(b) {
-		switch t.state {
-		case ruleLTRFinal, ruleInitial:
-			return bidi.LeftToRight, nil
-		case ruleRTLFinal:
-			return bidi.RightToLeft, nil
+// From RFC 5893
+// An RTL label is a label that contains at least one character of type
+// R, AL, or AN.
+//
+// An LTR label is any label that is not an RTL label.
+
+// Direction reports the direction of the given label as defined by RFC 5893.
+// The Bidi Rule does not have to be applied to labels of the category
+// LeftToRight.
+func Direction(b []byte) bidi.Direction {
+	for i := 0; i < len(b); {
+		e, sz := bidi.Lookup(b[i:])
+		if sz == 0 {
+			i++
 		}
+		c := e.Class()
+		if c == bidi.R || c == bidi.AL || c == bidi.AN {
+			return bidi.RightToLeft
+		}
+		i += sz
 	}
-	return bidi.Neutral, ErrInvalid
+	return bidi.LeftToRight
 }
 
 // DirectionString reports the direction of the given label as defined by RFC
-// 5893 or an error if s is not a valid label according to the Bidi Rule.
-func DirectionString(s string) (bidi.Direction, error) {
-	t := Transformer{}
-	if n, ok := t.advanceString(s); ok && n == len(s) {
-		switch t.state {
-		case ruleLTRFinal, ruleInitial:
-			return bidi.LeftToRight, nil
-		case ruleRTLFinal:
-			return bidi.RightToLeft, nil
+// 5893. The Bidi Rule does not have to be applied to labels of the category
+// LeftToRight.
+func DirectionString(s string) bidi.Direction {
+	for i := 0; i < len(s); {
+		e, sz := bidi.LookupString(s[i:])
+		if sz == 0 {
+			i++
 		}
+		c := e.Class()
+		if c == bidi.R || c == bidi.AL || c == bidi.AN {
+			return bidi.RightToLeft
+		}
+		i += sz
 	}
-	return bidi.Neutral, ErrInvalid
+	return bidi.LeftToRight
+}
+
+// Valid reports whether b conforms to the BiDi rule.
+func Valid(b []byte) bool {
+	var t Transformer
+	if n, ok := t.advance(b); !ok || n < len(b) {
+		return false
+	}
+	return t.isFinal()
+}
+
+// ValidString reports whether s conforms to the BiDi rule.
+func ValidString(s string) bool {
+	var t Transformer
+	if n, ok := t.advanceString(s); !ok || n < len(s) {
+		return false
+	}
+	return t.isFinal()
 }
 
 // New returns a Transformer that verifies that input adheres to the Bidi Rule.
@@ -160,8 +190,23 @@ func New() *Transformer {
 
 // Transformer implements transform.Transform.
 type Transformer struct {
-	state ruleState
-	seen  uint16
+	state  ruleState
+	hasRTL bool
+	seen   uint16
+}
+
+// A rule can only be violated for "Bidi Domain names", meaning if one of the
+// following categories has been observed.
+func (t *Transformer) isRTL() bool {
+	const isRTL = 1<<bidi.R | 1<<bidi.AL | 1<<bidi.AN
+	return t.seen&isRTL != 0
+}
+
+func (t *Transformer) isFinal() bool {
+	if !t.isRTL() {
+		return true
+	}
+	return t.state == ruleLTRFinal || t.state == ruleRTLFinal || t.state == ruleInitial
 }
 
 // Reset implements transform.Transformer.
@@ -185,7 +230,7 @@ func (t *Transformer) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, er
 
 // Span returns the first n bytes of src that conform to the Bidi rule.
 func (t *Transformer) Span(src []byte, atEOF bool) (n int, err error) {
-	if t.state == ruleInvalid {
+	if t.state == ruleInvalid && t.isRTL() {
 		return 0, ErrInvalid
 	}
 	n, ok := t.advance(src)
@@ -198,7 +243,7 @@ func (t *Transformer) Span(src []byte, atEOF bool) (n int, err error) {
 			break
 		}
 		err = ErrInvalid
-	case t.state != ruleLTRFinal && t.state != ruleRTLFinal && t.state != ruleInitial:
+	case !t.isFinal():
 		err = ErrInvalid
 	}
 	return n, err
@@ -225,12 +270,15 @@ func (t *Transformer) advance(s []byte) (n int, ok bool) {
 			e, sz = bidi.Lookup(s[n:])
 			if sz <= 1 {
 				if sz == 1 {
-					return n, false // invalid UTF-8
+					// We always consider invalid UTF-8 to be invalid, even if
+					// the string has not yet been determined to be RTL.
+					// TODO: is this correct?
+					return n, false
 				}
 				return n, true // incomplete UTF-8 encoding
 			}
 		}
-		// TODO: using CompactClass results in noticeable speedup.
+		// TODO: using CompactClass would result in noticeable speedup.
 		// See unicode/bidi/prop.go:Properties.CompactClass.
 		c := uint16(1 << e.Class())
 		t.seen |= c
@@ -245,7 +293,9 @@ func (t *Transformer) advance(s []byte) (n int, ok bool) {
 			t.state = tr[1].next
 		default:
 			t.state = ruleInvalid
-			return n, false
+			if t.isRTL() {
+				return n, false
+			}
 		}
 		n += sz
 	}
@@ -282,7 +332,9 @@ func (t *Transformer) advanceString(s string) (n int, ok bool) {
 			t.state = tr[1].next
 		default:
 			t.state = ruleInvalid
-			return n, false
+			if t.isRTL() {
+				return n, false
+			}
 		}
 		n += sz
 	}

vendor/golang.org/x/text/secure/precis/options.go

@@ -6,10 +6,10 @@ package precis
 
 import (
 	"golang.org/x/text/cases"
+	"golang.org/x/text/language"
 	"golang.org/x/text/runes"
 	"golang.org/x/text/transform"
 	"golang.org/x/text/unicode/norm"
-	"golang.org/x/text/width"
 )
 
 // An Option is used to define the behavior and rules of a Profile.
@@ -20,11 +20,12 @@ type options struct {
 	foldWidth bool
 
 	// Enforcement options
-	cases         transform.Transformer
+	asciiLower    bool
+	cases         transform.SpanningTransformer
 	disallow      runes.Set
-	norm          norm.Form
-	additional    []func() transform.Transformer
-	width         *width.Transformer
+	norm          transform.SpanningTransformer
+	additional    []func() transform.SpanningTransformer
+	width         transform.SpanningTransformer
 	disallowEmpty bool
 	bidiRule      bool
 
@@ -36,6 +37,11 @@ func getOpts(o ...Option) (res options) {
 	for _, f := range o {
 		f(&res)
 	}
+	// Using a SpanningTransformer, instead of norm.Form prevents an allocation
+	// down the road.
+	if res.norm == nil {
+		res.norm = norm.NFC
+	}
 	return
 }
 
@@ -74,11 +80,36 @@ var (
 	}
 )
 
+// TODO: move this logic to package transform
+
+type spanWrap struct{ transform.Transformer }
+
+func (s spanWrap) Span(src []byte, atEOF bool) (n int, err error) {
+	return 0, transform.ErrEndOfSpan
+}
+
+// TODO: allow different types? For instance:
+//     func() transform.Transformer
+//     func() transform.SpanningTransformer
+//     func([]byte) bool  // validation only
+//
+// Also, would be great if we could detect if a transformer is reentrant.
+
 // The AdditionalMapping option defines the additional mapping rule for the
 // Profile by applying Transformer's in sequence.
 func AdditionalMapping(t ...func() transform.Transformer) Option {
 	return func(o *options) {
-		o.additional = t
+		for _, f := range t {
+			sf := func() transform.SpanningTransformer {
+				return f().(transform.SpanningTransformer)
+			}
+			if _, ok := f().(transform.SpanningTransformer); !ok {
+				sf = func() transform.SpanningTransformer {
+					return spanWrap{f()}
+				}
+			}
+			o.additional = append(o.additional, sf)
+		}
 	}
 }
 
@@ -93,10 +124,26 @@ func Norm(f norm.Form) Option {
 // provided to determine the type of case folding used.
 func FoldCase(opts ...cases.Option) Option {
 	return func(o *options) {
+		o.asciiLower = true
 		o.cases = cases.Fold(opts...)
 	}
 }
 
+// The LowerCase option defines a Profile's case mapping rule. Options can be
+// provided to determine the type of case folding used.
+func LowerCase(opts ...cases.Option) Option {
+	return func(o *options) {
+		o.asciiLower = true
+		if len(opts) == 0 {
+			o.cases = cases.Lower(language.Und, cases.HandleFinalSigma(false))
+			return
+		}
+
+		opts = append([]cases.Option{cases.HandleFinalSigma(false)}, opts...)
+		o.cases = cases.Lower(language.Und, opts...)
+	}
+}
+
 // The Disallow option further restricts a Profile's allowed characters beyond
 // what is disallowed by the underlying string class.
 func Disallow(set runes.Set) Option {

vendor/golang.org/x/text/secure/precis/profile.go

@@ -5,9 +5,12 @@
 package precis
 
 import (
+	"bytes"
 	"errors"
 	"unicode/utf8"
 
+	"golang.org/x/text/cases"
+	"golang.org/x/text/language"
 	"golang.org/x/text/runes"
 	"golang.org/x/text/secure/bidirule"
 	"golang.org/x/text/transform"
@@ -90,32 +93,80 @@ type buffers struct {
 	next int
 }
 
-func (b *buffers) init(n int) {
-	b.buf[0] = make([]byte, 0, n)
-	b.buf[1] = make([]byte, 0, n)
-}
-
-func (b *buffers) apply(t transform.Transformer) (err error) {
-	// TODO: use Span, once available.
+func (b *buffers) apply(t transform.SpanningTransformer) (err error) {
+	n, err := t.Span(b.src, true)
+	if err != transform.ErrEndOfSpan {
+		return err
+	}
 	x := b.next & 1
-	b.src, _, err = transform.Append(t, b.buf[x][:0], b.src)
+	if b.buf[x] == nil {
+		b.buf[x] = make([]byte, 0, 8+len(b.src)+len(b.src)>>2)
+	}
+	span := append(b.buf[x][:0], b.src[:n]...)
+	b.src, _, err = transform.Append(t, span, b.src[n:])
 	b.buf[x] = b.src
 	b.next++
 	return err
 }
 
-func (b *buffers) enforce(p *Profile, src []byte) (str []byte, err error) {
+// Pre-allocate transformers when possible. In some cases this avoids allocation.
+var (
+	foldWidthT transform.SpanningTransformer = width.Fold
+	lowerCaseT transform.SpanningTransformer = cases.Lower(language.Und, cases.HandleFinalSigma(false))
+)
+
+// TODO: make this a method on profile.
+
+func (b *buffers) enforce(p *Profile, src []byte, comparing bool) (str []byte, err error) {
 	b.src = src
 
+	ascii := true
+	for _, c := range src {
+		if c >= utf8.RuneSelf {
+			ascii = false
+			break
+		}
+	}
+	// ASCII fast path.
+	if ascii {
+		for _, f := range p.options.additional {
+			if err = b.apply(f()); err != nil {
+				return nil, err
+			}
+		}
+		switch {
+		case p.options.asciiLower || (comparing && p.options.ignorecase):
+			for i, c := range b.src {
+				if 'A' <= c && c <= 'Z' {
+					b.src[i] = c ^ 1<<5
+				}
+			}
+		case p.options.cases != nil:
+			b.apply(p.options.cases)
+		}
+		c := checker{p: p}
+		if _, err := c.span(b.src, true); err != nil {
+			return nil, err
+		}
+		if p.disallow != nil {
+			for _, c := range b.src {
+				if p.disallow.Contains(rune(c)) {
+					return nil, errDisallowedRune
+				}
+			}
+		}
+		if p.options.disallowEmpty && len(b.src) == 0 {
+			return nil, errEmptyString
+		}
+		return b.src, nil
+	}
+
 	// These transforms are applied in the order defined in
 	// https://tools.ietf.org/html/rfc7564#section-7
 
 	// TODO: allow different width transforms options.
-	if p.options.foldWidth {
-		// TODO: use Span, once available.
-		if err = b.apply(width.Fold); err != nil {
-			return nil, err
-		}
+	if p.options.foldWidth || (p.options.ignorecase && comparing) {
+		b.apply(foldWidthT)
 	}
 	for _, f := range p.options.additional {
 		if err = b.apply(f()); err != nil {
@@ -123,24 +174,14 @@ func (b *buffers) enforce(p *Profile, src []byte) (str []byte, err error) {
 		}
 	}
 	if p.options.cases != nil {
-		if err = b.apply(p.options.cases); err != nil {
-			return nil, err
-		}
+		b.apply(p.options.cases)
 	}
-	if n := p.norm.QuickSpan(b.src); n < len(b.src) {
-		x := b.next & 1
-		n = copy(b.buf[x], b.src[:n])
-		b.src, _, err = transform.Append(p.norm, b.buf[x][:n], b.src[n:])
-		b.buf[x] = b.src
-		b.next++
-		if err != nil {
-			return nil, err
-		}
+	if comparing && p.options.ignorecase {
+		b.apply(lowerCaseT)
 	}
-	if p.options.bidiRule {
-		if err := b.apply(bidirule.New()); err != nil {
-			return nil, err
-		}
+	b.apply(p.norm)
+	if p.options.bidiRule && !bidirule.Valid(b.src) {
+		return nil, bidirule.ErrInvalid
 	}
 	c := checker{p: p}
 	if _, err := c.span(b.src, true); err != nil {
@@ -155,9 +196,6 @@ func (b *buffers) enforce(p *Profile, src []byte) (str []byte, err error) {
 			i += size
 		}
 	}
-
-	// TODO: Add the disallow empty rule with a dummy transformer?
-
 	if p.options.disallowEmpty && len(b.src) == 0 {
 		return nil, errEmptyString
 	}
@@ -168,19 +206,16 @@ func (b *buffers) enforce(p *Profile, src []byte) (str []byte, err error) {
 // It returns an error if the input string is invalid.
 func (p *Profile) Append(dst, src []byte) ([]byte, error) {
 	var buf buffers
-	buf.init(8 + len(src) + len(src)>>2)
-	b, err := buf.enforce(p, src)
+	b, err := buf.enforce(p, src, false)
 	if err != nil {
 		return nil, err
 	}
 	return append(dst, b...), nil
 }
 
-// Bytes returns a new byte slice with the result of applying the profile to b.
-func (p *Profile) Bytes(b []byte) ([]byte, error) {
+func processBytes(p *Profile, b []byte, key bool) ([]byte, error) {
 	var buf buffers
-	buf.init(8 + len(b) + len(b)>>2)
-	b, err := buf.enforce(p, b)
+	b, err := buf.enforce(p, b, key)
 	if err != nil {
 		return nil, err
 	}
@@ -192,39 +227,62 @@ func (p *Profile) Bytes(b []byte) ([]byte, error) {
 	return b, nil
 }
 
-// String returns a string with the result of applying the profile to s.
-func (p *Profile) String(s string) (string, error) {
+// Bytes returns a new byte slice with the result of applying the profile to b.
+func (p *Profile) Bytes(b []byte) ([]byte, error) {
+	return processBytes(p, b, false)
+}
+
+// AppendCompareKey appends the result of applying p to src (including any
+// optional rules to make strings comparable or useful in a map key such as
+// applying lowercasing) writing the result to dst. It returns an error if the
+// input string is invalid.
+func (p *Profile) AppendCompareKey(dst, src []byte) ([]byte, error) {
 	var buf buffers
-	buf.init(8 + len(s) + len(s)>>2)
-	b, err := buf.enforce(p, []byte(s))
+	b, err := buf.enforce(p, src, true)
+	if err != nil {
+		return nil, err
+	}
+	return append(dst, b...), nil
+}
+
+func processString(p *Profile, s string, key bool) (string, error) {
+	var buf buffers
+	b, err := buf.enforce(p, []byte(s), key)
 	if err != nil {
 		return "", err
 	}
 	return string(b), nil
 }
 
+// String returns a string with the result of applying the profile to s.
+func (p *Profile) String(s string) (string, error) {
+	return processString(p, s, false)
+}
+
+// CompareKey returns a string that can be used for comparison, hashing, or
+// collation.
+func (p *Profile) CompareKey(s string) (string, error) {
+	return processString(p, s, true)
+}
+
 // Compare enforces both strings, and then compares them for bit-string identity
 // (byte-for-byte equality). If either string cannot be enforced, the comparison
 // is false.
 func (p *Profile) Compare(a, b string) bool {
-	a, err := p.String(a)
-	if err != nil {
-		return false
-	}
-	b, err = p.String(b)
+	var buf buffers
+
+	akey, err := buf.enforce(p, []byte(a), true)
 	if err != nil {
 		return false
 	}
 
-	// TODO: This is out of order. Need to extract the transformation logic and
-	// put this in where the normal case folding would go (but only for
-	// comparison).
-	if p.options.ignorecase {
-		a = width.Fold.String(a)
-		b = width.Fold.String(a)
+	buf = buffers{}
+	bkey, err := buf.enforce(p, []byte(b), true)
+	if err != nil {
+		return false
 	}
 
-	return a == b
+	return bytes.Compare(akey, bkey) == 0
 }
 
 // Allowed returns a runes.Set containing every rune that is a member of the
@@ -264,33 +322,35 @@ func (c *checker) span(src []byte, atEOF bool) (n int, err error) {
 			}
 			return n, errDisallowedRune
 		}
+		doLookAhead := false
 		if property(e) < c.p.class.validFrom {
 			if d.rule == nil {
 				return n, errDisallowedRune
 			}
-			doLookAhead, err := d.rule(c.beforeBits)
+			doLookAhead, err = d.rule(c.beforeBits)
 			if err != nil {
 				return n, err
 			}
-			if doLookAhead {
-				c.beforeBits &= d.keep
-				c.beforeBits |= d.set
-				// We may still have a lookahead rule which we will require to
-				// complete (by checking termBits == 0) before setting the new
-				// bits.
-				if c.termBits != 0 && (!c.checkLookahead() || c.termBits == 0) {
-					return n, err
-				}
-				c.termBits = d.term
-				c.acceptBits = d.accept
-				n += sz
-				continue
-			}
 		}
 		c.beforeBits &= d.keep
 		c.beforeBits |= d.set
-		if c.termBits != 0 && !c.checkLookahead() {
-			return n, errContext
+		if c.termBits != 0 {
+			// We are currently in an unterminated lookahead.
+			if c.beforeBits&c.termBits != 0 {
+				c.termBits = 0
+				c.acceptBits = 0
+			} else if c.beforeBits&c.acceptBits == 0 {
+				// Invalid continuation of the unterminated lookahead sequence.
+				return n, errContext
+			}
+		}
+		if doLookAhead {
+			if c.termBits != 0 {
+				// A previous lookahead run has not been terminated yet.
+				return n, errContext
+			}
+			c.termBits = d.term
+			c.acceptBits = d.accept
 		}
 		n += sz
 	}
@@ -300,18 +360,6 @@ func (c *checker) span(src []byte, atEOF bool) (n int, err error) {
 	return n, err
 }
 
-func (c *checker) checkLookahead() bool {
-	switch {
-	case c.beforeBits&c.termBits != 0:
-		c.termBits = 0
-		c.acceptBits = 0
-	case c.beforeBits&c.acceptBits != 0:
-	default:
-		return false
-	}
-	return true
-}
-
 // TODO: we may get rid of this transform if transform.Chain understands
 // something like a Spanner interface.
 func (c checker) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {

vendor/golang.org/x/text/secure/precis/profiles.go

@@ -13,44 +13,66 @@ import (
 )
 
 var (
-	Nickname              *Profile = nickname          // Implements the Nickname profile specified in RFC 7700.
-	UsernameCaseMapped    *Profile = usernameCaseMap   // Implements the UsernameCaseMapped profile specified in RFC 7613.
-	UsernameCasePreserved *Profile = usernameNoCaseMap // Implements the UsernameCasePreserved profile specified in RFC 7613.
-	OpaqueString          *Profile = opaquestring      // Implements the OpaqueString profile defined in RFC 7613 for passwords and other secure labels.
+	// Implements the Nickname profile specified in RFC 7700.
+	// The nickname profile is not idempotent and may need to be applied multiple
+	// times before being used for comparisons.
+	Nickname *Profile = nickname
+
+	// Implements the UsernameCaseMapped profile specified in RFC 7613.
+	UsernameCaseMapped *Profile = usernameCaseMap
+
+	// Implements the UsernameCasePreserved profile specified in RFC 7613.
+	UsernameCasePreserved *Profile = usernameNoCaseMap
+
+	// Implements the OpaqueString profile defined in RFC 7613 for passwords and other secure labels.
+	OpaqueString *Profile = opaquestring
 )
 
-// TODO: mvl: "Ultimately, I would manually define the structs for the internal
-// profiles. This avoid pulling in unneeded tables when they are not used."
 var (
-	nickname = NewFreeform(
-		AdditionalMapping(func() transform.Transformer {
-			return &nickAdditionalMapping{}
-		}),
-		IgnoreCase,
-		Norm(norm.NFKC),
-		DisallowEmpty,
-	)
-	usernameCaseMap = NewIdentifier(
-		FoldWidth,
-		FoldCase(),
-		Norm(norm.NFC),
-		BidiRule,
-	)
-	usernameNoCaseMap = NewIdentifier(
-		FoldWidth,
-		Norm(norm.NFC),
-		BidiRule,
-	)
-	opaquestring = NewFreeform(
-		AdditionalMapping(func() transform.Transformer {
-			return runes.Map(func(r rune) rune {
-				if unicode.Is(unicode.Zs, r) {
-					return ' '
-				}
-				return r
-			})
-		}),
-		Norm(norm.NFC),
-		DisallowEmpty,
-	)
+	nickname = &Profile{
+		options: getOpts(
+			AdditionalMapping(func() transform.Transformer {
+				return &nickAdditionalMapping{}
+			}),
+			IgnoreCase,
+			Norm(norm.NFKC),
+			DisallowEmpty,
+		),
+		class: freeform,
+	}
+	usernameCaseMap = &Profile{
+		options: getOpts(
+			FoldWidth,
+			LowerCase(),
+			Norm(norm.NFC),
+			BidiRule,
+		),
+		class: identifier,
+	}
+	usernameNoCaseMap = &Profile{
+		options: getOpts(
+			FoldWidth,
+			Norm(norm.NFC),
+			BidiRule,
+		),
+		class: identifier,
+	}
+	opaquestring = &Profile{
+		options: getOpts(
+			AdditionalMapping(func() transform.Transformer {
+				return mapSpaces
+			}),
+			Norm(norm.NFC),
+			DisallowEmpty,
+		),
+		class: freeform,
+	}
 )
+
+// mapSpaces is a shared value of a runes.Map transformer.
+var mapSpaces transform.Transformer = runes.Map(func(r rune) rune {
+	if unicode.Is(unicode.Zs, r) {
+		return ' '
+	}
+	return r
+})

vendor/golang.org/x/text/secure/precis/tables.go

@@ -1,4 +1,4 @@
-// This file was generated by go generate; DO NOT EDIT
+// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
 
 package precis
 

vendor/golang.org/x/text/secure/precis/trieval.go

@@ -1,4 +1,4 @@
-// This file was generated by go generate; DO NOT EDIT
+// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
 
 package precis
 

vendor/golang.org/x/text/transform/transform.go

@@ -24,6 +24,10 @@ var (
 	// complete the transformation.
 	ErrShortSrc = errors.New("transform: short source buffer")
 
+	// ErrEndOfSpan means that the input and output (the transformed input)
+	// are not identical.
+	ErrEndOfSpan = errors.New("transform: input and output are not identical")
+
 	// errInconsistentByteCount means that Transform returned success (nil
 	// error) but also returned nSrc inconsistent with the src argument.
 	errInconsistentByteCount = errors.New("transform: inconsistent byte count returned")
@@ -60,6 +64,41 @@ type Transformer interface {
 	Reset()
 }
 
+// SpanningTransformer extends the Transformer interface with a Span method
+// that determines how much of the input already conforms to the Transformer.
+type SpanningTransformer interface {
+	Transformer
+
+	// Span returns a position in src such that transforming src[:n] results in
+	// identical output src[:n] for these bytes. It does not necessarily return
+	// the largest such n. The atEOF argument tells whether src represents the
+	// last bytes of the input.
+	//
+	// Callers should always account for the n bytes consumed before
+	// considering the error err.
+	//
+	// A nil error means that all input bytes are known to be identical to the
+	// output produced by the Transformer. A nil error can be be returned
+	// regardless of whether atEOF is true. If err is nil, then then n must
+	// equal len(src); the converse is not necessarily true.
+	//
+	// ErrEndOfSpan means that the Transformer output may differ from the
+	// input after n bytes. Note that n may be len(src), meaning that the output
+	// would contain additional bytes after otherwise identical output.
+	// ErrShortSrc means that src had insufficient data to determine whether the
+	// remaining bytes would change. Other than the error conditions listed
+	// here, implementations are free to report other errors that arise.
+	//
+	// Calling Span can modify the Transformer state as a side effect. In
+	// effect, it does the transformation just as calling Transform would, only
+	// without copying to a destination buffer and only up to a point it can
+	// determine the input and output bytes are the same. This is obviously more
+	// limited than calling Transform, but can be more efficient in terms of
+	// copying and allocating buffers. Calls to Span and Transform may be
+	// interleaved.
+	Span(src []byte, atEOF bool) (n int, err error)
+}
+
 // NopResetter can be embedded by implementations of Transformer to add a nop
 // Reset method.
 type NopResetter struct{}
@@ -278,6 +317,10 @@ func (nop) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
 	return n, n, err
 }
 
+func (nop) Span(src []byte, atEOF bool) (n int, err error) {
+	return len(src), nil
+}
+
 type discard struct{ NopResetter }
 
 func (discard) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
@@ -289,8 +332,8 @@ var (
 	// by consuming all bytes and writing nothing.
 	Discard Transformer = discard{}
 
-	// Nop is a Transformer that copies src to dst.
-	Nop Transformer = nop{}
+	// Nop is a SpanningTransformer that copies src to dst.
+	Nop SpanningTransformer = nop{}
 )
 
 // chain is a sequence of links. A chain with N Transformers has N+1 links and
@@ -358,6 +401,8 @@ func (c *chain) Reset() {
 	}
 }
 
+// TODO: make chain use Span (is going to be fun to implement!)
+
 // Transform applies the transformers of c in sequence.
 func (c *chain) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
 	// Set up src and dst in the chain.
@@ -448,8 +493,7 @@ func (c *chain) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err erro
 	return dstL.n, srcL.p, err
 }
 
-// RemoveFunc returns a Transformer that removes from the input all runes r for
-// which f(r) is true. Illegal bytes in the input are replaced by RuneError.
+// Deprecated: use runes.Remove instead.
 func RemoveFunc(f func(r rune) bool) Transformer {
 	return removeF(f)
 }

vendor/golang.org/x/text/unicode/bidi/bracket.go

@@ -84,7 +84,7 @@ func resolvePairedBrackets(s *isolatingRunSequence) {
 		dirEmbed = R
 	}
 	p.locateBrackets(s.p.pairTypes, s.p.pairValues)
-	p.resolveBrackets(dirEmbed)
+	p.resolveBrackets(dirEmbed, s.p.initialTypes)
 }
 
 type bracketPairer struct {
@@ -125,6 +125,8 @@ func (p *bracketPairer) matchOpener(pairValues []rune, opener, closer int) bool
 	return pairValues[p.indexes[opener]] == pairValues[p.indexes[closer]]
 }
 
+const maxPairingDepth = 63
+
 // locateBrackets locates matching bracket pairs according to BD16.
 //
 // This implementation uses a linked list instead of a stack, because, while
@@ -136,11 +138,17 @@ func (p *bracketPairer) locateBrackets(pairTypes []bracketType, pairValues []run
 	for i, index := range p.indexes {
 
 		// look at the bracket type for each character
-		switch pairTypes[index] {
-		case bpNone:
+		if pairTypes[index] == bpNone || p.codesIsolatedRun[i] != ON {
 			// continue scanning
-
+			continue
+		}
+		switch pairTypes[index] {
 		case bpOpen:
+			// check if maximum pairing depth reached
+			if p.openers.Len() == maxPairingDepth {
+				p.openers.Init()
+				return
+			}
 			// remember opener location, most recent first
 			p.openers.PushFront(i)
 
@@ -270,7 +278,7 @@ func (p *bracketPairer) classBeforePair(loc bracketPair) Class {
 }
 
 // assignBracketType implements rule N0 for a single bracket pair.
-func (p *bracketPairer) assignBracketType(loc bracketPair, dirEmbed Class) {
+func (p *bracketPairer) assignBracketType(loc bracketPair, dirEmbed Class, initialTypes []Class) {
 	// rule "N0, a", inspect contents of pair
 	dirPair := p.classifyPairContent(loc, dirEmbed)
 
@@ -295,13 +303,33 @@ func (p *bracketPairer) assignBracketType(loc bracketPair, dirEmbed Class) {
 	// direction
 
 	// set the bracket types to the type found
+	p.setBracketsToType(loc, dirPair, initialTypes)
+}
+
+func (p *bracketPairer) setBracketsToType(loc bracketPair, dirPair Class, initialTypes []Class) {
 	p.codesIsolatedRun[loc.opener] = dirPair
 	p.codesIsolatedRun[loc.closer] = dirPair
+
+	for i := loc.opener + 1; i < loc.closer; i++ {
+		index := p.indexes[i]
+		if initialTypes[index] != NSM {
+			break
+		}
+		p.codesIsolatedRun[i] = dirPair
+	}
+
+	for i := loc.closer + 1; i < len(p.indexes); i++ {
+		index := p.indexes[i]
+		if initialTypes[index] != NSM {
+			break
+		}
+		p.codesIsolatedRun[i] = dirPair
+	}
 }
 
 // resolveBrackets implements rule N0 for a list of pairs.
-func (p *bracketPairer) resolveBrackets(dirEmbed Class) {
+func (p *bracketPairer) resolveBrackets(dirEmbed Class, initialTypes []Class) {
 	for _, loc := range p.pairPositions {
-		p.assignBracketType(loc, dirEmbed)
+		p.assignBracketType(loc, dirEmbed, initialTypes)
 	}
 }

vendor/golang.org/x/text/unicode/bidi/core.go

@@ -309,6 +309,9 @@ func (p *paragraph) determineExplicitEmbeddingLevels() {
 			}
 			if isIsolate {
 				p.resultLevels[i] = stack.lastEmbeddingLevel()
+				if stack.lastDirectionalOverrideStatus() != ON {
+					p.resultTypes[i] = stack.lastDirectionalOverrideStatus()
+				}
 			}
 
 			var newLevel level
@@ -723,7 +726,7 @@ loop:
 				continue loop
 			}
 		}
-		log.Panicf("invalid bidi code %s present in assertOnly at position %d", t, s.indexes[i])
+		log.Panicf("invalid bidi code %v present in assertOnly at position %d", t, s.indexes[i])
 	}
 }
 

vendor/golang.org/x/text/unicode/bidi/tables.go

@@ -1,4 +1,4 @@
-// This file was generated by go generate; DO NOT EDIT
+// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
 
 package bidi
 

vendor/golang.org/x/text/unicode/bidi/trieval.go

@@ -1,4 +1,4 @@
-// This file was generated by go generate; DO NOT EDIT
+// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
 
 package bidi
 

vendor/golang.org/x/text/unicode/norm/composition.go

@@ -33,17 +33,9 @@ const (
 // streamSafe implements the policy of when a CGJ should be inserted.
 type streamSafe uint8
 
-// mkStreamSafe is a shorthand for declaring a streamSafe var and calling
-// first on it.
-func mkStreamSafe(p Properties) streamSafe {
-	return streamSafe(p.nTrailingNonStarters())
-}
-
-// first inserts the first rune of a segment.
+// first inserts the first rune of a segment. It is a faster version of next if
+// it is known p represents the first rune in a segment.
 func (ss *streamSafe) first(p Properties) {
-	if *ss != 0 {
-		panic("!= 0")
-	}
 	*ss = streamSafe(p.nTrailingNonStarters())
 }
 
@@ -66,7 +58,7 @@ func (ss *streamSafe) next(p Properties) ssState {
 	// be a non-starter. Note that it always hold that if nLead > 0 then
 	// nLead == nTrail.
 	if n == 0 {
-		*ss = 0
+		*ss = streamSafe(p.nTrailingNonStarters())
 		return ssStarter
 	}
 	return ssSuccess
@@ -142,7 +134,6 @@ func (rb *reorderBuffer) setFlusher(out []byte, f func(*reorderBuffer) bool) {
 func (rb *reorderBuffer) reset() {
 	rb.nrune = 0
 	rb.nbyte = 0
-	rb.ss = 0
 }
 
 func (rb *reorderBuffer) doFlush() bool {
@@ -257,6 +248,9 @@ func (rb *reorderBuffer) insertUnsafe(src input, i int, info Properties) {
 // It flushes the buffer on each new segment start.
 func (rb *reorderBuffer) insertDecomposed(dcomp []byte) insertErr {
 	rb.tmpBytes.setBytes(dcomp)
+	// As the streamSafe accounting already handles the counting for modifiers,
+	// we don't have to call next. However, we do need to keep the accounting
+	// intact when flushing the buffer.
 	for i := 0; i < len(dcomp); {
 		info := rb.f.info(rb.tmpBytes, i)
 		if info.BoundaryBefore() && rb.nrune > 0 && !rb.doFlush() {

vendor/golang.org/x/text/unicode/norm/forminfo.go

@@ -10,7 +10,7 @@ package norm
 // and its corresponding decomposing form share the same trie.  Each trie maps
 // a rune to a uint16. The values take two forms.  For v >= 0x8000:
 //   bits
-//   15:    1 (inverse of NFD_QD bit of qcInfo)
+//   15:    1 (inverse of NFD_QC bit of qcInfo)
 //   13..7: qcInfo (see below). isYesD is always true (no decompostion).
 //    6..0: ccc (compressed CCC value).
 // For v < 0x8000, the respective rune has a decomposition and v is an index
@@ -56,28 +56,31 @@ type formInfo struct {
 	nextMain                 iterFunc
 }
 
-var formTable []*formInfo
-
-func init() {
-	formTable = make([]*formInfo, 4)
-
-	for i := range formTable {
-		f := &formInfo{}
-		formTable[i] = f
-		f.form = Form(i)
-		if Form(i) == NFKD || Form(i) == NFKC {
-			f.compatibility = true
-			f.info = lookupInfoNFKC
-		} else {
-			f.info = lookupInfoNFC
-		}
-		f.nextMain = nextDecomposed
-		if Form(i) == NFC || Form(i) == NFKC {
-			f.nextMain = nextComposed
-			f.composing = true
-		}
-	}
-}
+var formTable = []*formInfo{{
+	form:          NFC,
+	composing:     true,
+	compatibility: false,
+	info:          lookupInfoNFC,
+	nextMain:      nextComposed,
+}, {
+	form:          NFD,
+	composing:     false,
+	compatibility: false,
+	info:          lookupInfoNFC,
+	nextMain:      nextDecomposed,
+}, {
+	form:          NFKC,
+	composing:     true,
+	compatibility: true,
+	info:          lookupInfoNFKC,
+	nextMain:      nextComposed,
+}, {
+	form:          NFKD,
+	composing:     false,
+	compatibility: true,
+	info:          lookupInfoNFKC,
+	nextMain:      nextDecomposed,
+}}
 
 // We do not distinguish between boundaries for NFC, NFD, etc. to avoid
 // unexpected behavior for the user.  For example, in NFD, there is a boundary

vendor/golang.org/x/text/unicode/norm/input.go

@@ -90,16 +90,20 @@ func (in *input) charinfoNFKC(p int) (uint16, int) {
 }
 
 func (in *input) hangul(p int) (r rune) {
+	var size int
 	if in.bytes == nil {
 		if !isHangulString(in.str[p:]) {
 			return 0
 		}
-		r, _ = utf8.DecodeRuneInString(in.str[p:])
+		r, size = utf8.DecodeRuneInString(in.str[p:])
 	} else {
 		if !isHangul(in.bytes[p:]) {
 			return 0
 		}
-		r, _ = utf8.DecodeRune(in.bytes[p:])
+		r, size = utf8.DecodeRune(in.bytes[p:])
+	}
+	if size != hangulUTF8Size {
+		return 0
 	}
 	return r
 }

vendor/golang.org/x/text/unicode/norm/iter.go

@@ -41,6 +41,7 @@ func (i *Iter) Init(f Form, src []byte) {
 	i.next = i.rb.f.nextMain
 	i.asciiF = nextASCIIBytes
 	i.info = i.rb.f.info(i.rb.src, i.p)
+	i.rb.ss.first(i.info)
 }
 
 // InitString initializes i to iterate over src after normalizing it to Form f.
@@ -56,11 +57,12 @@ func (i *Iter) InitString(f Form, src string) {
 	i.next = i.rb.f.nextMain
 	i.asciiF = nextASCIIString
 	i.info = i.rb.f.info(i.rb.src, i.p)
+	i.rb.ss.first(i.info)
 }
 
 // Seek sets the segment to be returned by the next call to Next to start
 // at position p.  It is the responsibility of the caller to set p to the
-// start of a UTF8 rune.
+// start of a segment.
 func (i *Iter) Seek(offset int64, whence int) (int64, error) {
 	var abs int64
 	switch whence {
@@ -84,6 +86,7 @@ func (i *Iter) Seek(offset int64, whence int) (int64, error) {
 	i.multiSeg = nil
 	i.next = i.rb.f.nextMain
 	i.info = i.rb.f.info(i.rb.src, i.p)
+	i.rb.ss.first(i.info)
 	return abs, nil
 }
 
@@ -161,6 +164,7 @@ func nextHangul(i *Iter) []byte {
 	if next >= i.rb.nsrc {
 		i.setDone()
 	} else if i.rb.src.hangul(next) == 0 {
+		i.rb.ss.next(i.info)
 		i.info = i.rb.f.info(i.rb.src, i.p)
 		i.next = i.rb.f.nextMain
 		return i.next(i)
@@ -204,12 +208,10 @@ func nextMultiNorm(i *Iter) []byte {
 		if info.BoundaryBefore() {
 			i.rb.compose()
 			seg := i.buf[:i.rb.flushCopy(i.buf[:])]
-			i.rb.ss.first(info)
 			i.rb.insertUnsafe(input{bytes: d}, j, info)
 			i.multiSeg = d[j+int(info.size):]
 			return seg
 		}
-		i.rb.ss.next(info)
 		i.rb.insertUnsafe(input{bytes: d}, j, info)
 		j += int(info.size)
 	}
@@ -222,9 +224,9 @@ func nextMultiNorm(i *Iter) []byte {
 func nextDecomposed(i *Iter) (next []byte) {
 	outp := 0
 	inCopyStart, outCopyStart := i.p, 0
-	ss := mkStreamSafe(i.info)
 	for {
 		if sz := int(i.info.size); sz <= 1 {
+			i.rb.ss = 0
 			p := i.p
 			i.p++ // ASCII or illegal byte.  Either way, advance by 1.
 			if i.p >= i.rb.nsrc {
@@ -243,6 +245,8 @@ func nextDecomposed(i *Iter) (next []byte) {
 			p := outp + len(d)
 			if outp > 0 {
 				i.rb.src.copySlice(i.buf[outCopyStart:], inCopyStart, i.p)
+				// TODO: this condition should not be possible, but we leave it
+				// in for defensive purposes.
 				if p > len(i.buf) {
 					return i.buf[:outp]
 				}
@@ -266,7 +270,7 @@ func nextDecomposed(i *Iter) (next []byte) {
 			} else {
 				i.info = i.rb.f.info(i.rb.src, i.p)
 			}
-			switch ss.next(i.info) {
+			switch i.rb.ss.next(i.info) {
 			case ssOverflow:
 				i.next = nextCGJDecompose
 				fallthrough
@@ -309,7 +313,7 @@ func nextDecomposed(i *Iter) (next []byte) {
 		}
 		prevCC := i.info.tccc
 		i.info = i.rb.f.info(i.rb.src, i.p)
-		if v := ss.next(i.info); v == ssStarter {
+		if v := i.rb.ss.next(i.info); v == ssStarter {
 			break
 		} else if v == ssOverflow {
 			i.next = nextCGJDecompose
@@ -335,10 +339,6 @@ doNorm:
 
 func doNormDecomposed(i *Iter) []byte {
 	for {
-		if s := i.rb.ss.next(i.info); s == ssOverflow {
-			i.next = nextCGJDecompose
-			break
-		}
 		i.rb.insertUnsafe(i.rb.src, i.p, i.info)
 		if i.p += int(i.info.size); i.p >= i.rb.nsrc {
 			i.setDone()
@@ -348,6 +348,10 @@ func doNormDecomposed(i *Iter) []byte {
 		if i.info.ccc == 0 {
 			break
 		}
+		if s := i.rb.ss.next(i.info); s == ssOverflow {
+			i.next = nextCGJDecompose
+			break
+		}
 	}
 	// new segment or too many combining characters: exit normalization
 	return i.buf[:i.rb.flushCopy(i.buf[:])]
@@ -357,6 +361,7 @@ func nextCGJDecompose(i *Iter) []byte {
 	i.rb.ss = 0
 	i.rb.insertCGJ()
 	i.next = nextDecomposed
+	i.rb.ss.first(i.info)
 	buf := doNormDecomposed(i)
 	return buf
 }
@@ -365,7 +370,6 @@ func nextCGJDecompose(i *Iter) []byte {
 func nextComposed(i *Iter) []byte {
 	outp, startp := 0, i.p
 	var prevCC uint8
-	ss := mkStreamSafe(i.info)
 	for {
 		if !i.info.isYesC() {
 			goto doNorm
@@ -385,11 +389,12 @@ func nextComposed(i *Iter) []byte {
 			i.setDone()
 			break
 		} else if i.rb.src._byte(i.p) < utf8.RuneSelf {
+			i.rb.ss = 0
 			i.next = i.asciiF
 			break
 		}
 		i.info = i.rb.f.info(i.rb.src, i.p)
-		if v := ss.next(i.info); v == ssStarter {
+		if v := i.rb.ss.next(i.info); v == ssStarter {
 			break
 		} else if v == ssOverflow {
 			i.next = nextCGJCompose
@@ -401,8 +406,10 @@ func nextComposed(i *Iter) []byte {
 	}
 	return i.returnSlice(startp, i.p)
 doNorm:
+	// reset to start position
 	i.p = startp
 	i.info = i.rb.f.info(i.rb.src, i.p)
+	i.rb.ss.first(i.info)
 	if i.info.multiSegment() {
 		d := i.info.Decomposition()
 		info := i.rb.f.info(input{bytes: d}, 0)

vendor/golang.org/x/text/unicode/norm/maketables.go

@@ -35,12 +35,9 @@ func main() {
 	computeNonStarterCounts()
 	verifyComputed()
 	printChars()
-	if *test {
-		testDerived()
-		printTestdata()
-	} else {
-		makeTables()
-	}
+	testDerived()
+	printTestdata()
+	makeTables()
 }
 
 var (
@@ -602,6 +599,7 @@ func printCharInfoTables(w io.Writer) int {
 		}
 		index := normalDecomp
 		nTrail := chars[r].nTrailingNonStarters
+		nLead := chars[r].nLeadingNonStarters
 		if tccc > 0 || lccc > 0 || nTrail > 0 {
 			tccc <<= 2
 			tccc |= nTrail
@@ -612,7 +610,7 @@ func printCharInfoTables(w io.Writer) int {
 					index = firstCCC
 				}
 			}
-			if lccc > 0 {
+			if lccc > 0 || nLead > 0 {
 				s += string([]byte{lccc})
 				if index == firstCCC {
 					log.Fatalf("%U: multi-segment decomposition not supported for decompositions with leading CCC != 0", r)

vendor/golang.org/x/text/unicode/norm/normalize.go

@@ -2,13 +2,18 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+// Note: the file data_test.go that is generated should not be checked in.
 //go:generate go run maketables.go triegen.go
-//go:generate go run maketables.go triegen.go -test
+//go:generate go test -tags test
 
 // Package norm contains types and functions for normalizing Unicode strings.
 package norm
 
-import "unicode/utf8"
+import (
+	"unicode/utf8"
+
+	"golang.org/x/text/transform"
+)
 
 // A Form denotes a canonical representation of Unicode code points.
 // The Unicode-defined normalization and equivalence forms are:
@@ -263,6 +268,34 @@ func (f Form) QuickSpan(b []byte) int {
 	return n
 }
 
+// Span implements transform.SpanningTransformer. It returns a boundary n such
+// that b[0:n] == f(b[0:n]). It is not guaranteed to return the largest such n.
+func (f Form) Span(b []byte, atEOF bool) (n int, err error) {
+	n, ok := formTable[f].quickSpan(inputBytes(b), 0, len(b), atEOF)
+	if n < len(b) {
+		if !ok {
+			err = transform.ErrEndOfSpan
+		} else {
+			err = transform.ErrShortSrc
+		}
+	}
+	return n, err
+}
+
+// SpanString returns a boundary n such that s[0:n] == f(s[0:n]).
+// It is not guaranteed to return the largest such n.
+func (f Form) SpanString(s string, atEOF bool) (n int, err error) {
+	n, ok := formTable[f].quickSpan(inputString(s), 0, len(s), atEOF)
+	if n < len(s) {
+		if !ok {
+			err = transform.ErrEndOfSpan
+		} else {
+			err = transform.ErrShortSrc
+		}
+	}
+	return n, err
+}
+
 // quickSpan returns a boundary n such that src[0:n] == f(src[0:n]) and
 // whether any non-normalized parts were found. If atEOF is false, n will
 // not point past the last segment if this segment might be become
@@ -291,7 +324,6 @@ func (f *formInfo) quickSpan(src input, i, end int, atEOF bool) (n int, ok bool)
 		// have an overflow for runes that are starters (e.g. with U+FF9E).
 		switch ss.next(info) {
 		case ssStarter:
-			ss.first(info)
 			lastSegStart = i
 		case ssOverflow:
 			return lastSegStart, false
@@ -321,7 +353,7 @@ func (f *formInfo) quickSpan(src input, i, end int, atEOF bool) (n int, ok bool)
 	return lastSegStart, false
 }
 
-// QuickSpanString returns a boundary n such that b[0:n] == f(s[0:n]).
+// QuickSpanString returns a boundary n such that s[0:n] == f(s[0:n]).
 // It is not guaranteed to return the largest such n.
 func (f Form) QuickSpanString(s string) int {
 	n, _ := formTable[f].quickSpan(inputString(s), 0, len(s), true)
@@ -408,6 +440,8 @@ func (f Form) nextBoundary(src input, nsrc int, atEOF bool) int {
 			}
 			return -1
 		}
+		// TODO: Using streamSafe to determine the boundary isn't the same as
+		// using BoundaryBefore. Determine which should be used.
 		if s := ss.next(info); s != ssSuccess {
 			return i
 		}
@@ -472,15 +506,14 @@ func decomposeSegment(rb *reorderBuffer, sp int, atEOF bool) int {
 	if info.size == 0 {
 		return 0
 	}
-	if rb.nrune > 0 {
-		if s := rb.ss.next(info); s == ssStarter {
-			goto end
-		} else if s == ssOverflow {
-			rb.insertCGJ()
+	if s := rb.ss.next(info); s == ssStarter {
+		// TODO: this could be removed if we don't support merging.
+		if rb.nrune > 0 {
 			goto end
 		}
-	} else {
-		rb.ss.first(info)
+	} else if s == ssOverflow {
+		rb.insertCGJ()
+		goto end
 	}
 	if err := rb.insertFlush(rb.src, sp, info); err != iSuccess {
 		return int(err)

vendor/golang.org/x/text/unicode/norm/readwriter.go

@@ -112,7 +112,6 @@ func (r *normReader) Read(p []byte) (int, error) {
 			}
 		}
 	}
-	panic("should not reach here")
 }
 
 // Reader returns a new reader that implements Read

vendor/golang.org/x/text/width/kind_string.go

@@ -1,4 +1,4 @@
-// Code generated by "stringer -type=Kind"; DO NOT EDIT
+// Code generated by "stringer -type=Kind"; DO NOT EDIT.
 
 package width
 

vendor/golang.org/x/text/width/tables.go

@@ -1,4 +1,4 @@
-// This file was generated by go generate; DO NOT EDIT
+// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
 
 package width
 

vendor/golang.org/x/text/width/transform.go

@@ -14,6 +14,32 @@ type foldTransform struct {
 	transform.NopResetter
 }
 
+func (foldTransform) Span(src []byte, atEOF bool) (n int, err error) {
+	for n < len(src) {
+		if src[n] < utf8.RuneSelf {
+			// ASCII fast path.
+			for n++; n < len(src) && src[n] < utf8.RuneSelf; n++ {
+			}
+			continue
+		}
+		v, size := trie.lookup(src[n:])
+		if size == 0 { // incomplete UTF-8 encoding
+			if !atEOF {
+				err = transform.ErrShortSrc
+			} else {
+				n = len(src)
+			}
+			break
+		}
+		if elem(v)&tagNeedsFold != 0 {
+			err = transform.ErrEndOfSpan
+			break
+		}
+		n += size
+	}
+	return n, err
+}
+
 func (foldTransform) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
 	for nSrc < len(src) {
 		if src[nSrc] < utf8.RuneSelf {
@@ -70,6 +96,33 @@ type narrowTransform struct {
 	transform.NopResetter
 }
 
+func (narrowTransform) Span(src []byte, atEOF bool) (n int, err error) {
+	for n < len(src) {
+		if src[n] < utf8.RuneSelf {
+			// ASCII fast path.
+			for n++; n < len(src) && src[n] < utf8.RuneSelf; n++ {
+			}
+			continue
+		}
+		v, size := trie.lookup(src[n:])
+		if size == 0 { // incomplete UTF-8 encoding
+			if !atEOF {
+				err = transform.ErrShortSrc
+			} else {
+				n = len(src)
+			}
+			break
+		}
+		if k := elem(v).kind(); byte(v) == 0 || k != EastAsianFullwidth && k != EastAsianWide && k != EastAsianAmbiguous {
+		} else {
+			err = transform.ErrEndOfSpan
+			break
+		}
+		n += size
+	}
+	return n, err
+}
+
 func (narrowTransform) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
 	for nSrc < len(src) {
 		if src[nSrc] < utf8.RuneSelf {
@@ -126,6 +179,30 @@ type wideTransform struct {
 	transform.NopResetter
 }
 
+func (wideTransform) Span(src []byte, atEOF bool) (n int, err error) {
+	for n < len(src) {
+		// TODO: Consider ASCII fast path. Special-casing ASCII handling can
+		// reduce the ns/op of BenchmarkWideASCII by about 30%. This is probably
+		// not enough to warrant the extra code and complexity.
+		v, size := trie.lookup(src[n:])
+		if size == 0 { // incomplete UTF-8 encoding
+			if !atEOF {
+				err = transform.ErrShortSrc
+			} else {
+				n = len(src)
+			}
+			break
+		}
+		if k := elem(v).kind(); byte(v) == 0 || k != EastAsianHalfwidth && k != EastAsianNarrow {
+		} else {
+			err = transform.ErrEndOfSpan
+			break
+		}
+		n += size
+	}
+	return n, err
+}
+
 func (wideTransform) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
 	for nSrc < len(src) {
 		// TODO: Consider ASCII fast path. Special-casing ASCII handling can

vendor/golang.org/x/text/width/trieval.go

@@ -1,4 +1,4 @@
-// This file was generated by go generate; DO NOT EDIT
+// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
 
 package width
 

vendor/golang.org/x/text/width/width.go

@@ -153,17 +153,22 @@ func (p Properties) Wide() rune {
 
 // Transformer implements the transform.Transformer interface.
 type Transformer struct {
-	t transform.Transformer
+	t transform.SpanningTransformer
 }
 
 // Reset implements the transform.Transformer interface.
 func (t Transformer) Reset() { t.t.Reset() }
 
-// Transform implements the Transformer interface.
+// Transform implements the transform.Transformer interface.
 func (t Transformer) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
 	return t.t.Transform(dst, src, atEOF)
 }
 
+// Span implements the transform.SpanningTransformer interface.
+func (t Transformer) Span(src []byte, atEOF bool) (n int, err error) {
+	return t.t.Span(src, atEOF)
+}
+
 // Bytes returns a new byte slice with the result of applying t to b.
 func (t Transformer) Bytes(b []byte) []byte {
 	b, _, _ = transform.Bytes(t, b)