Update go dependencies (#4524)

vendor/golang.org/x/tools/go/gcexportdata/main.go

@@ -0,0 +1,99 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build ignore
+
+// The gcexportdata command is a diagnostic tool that displays the
+// contents of gc export data files.
+package main
+
+import (
+	"flag"
+	"fmt"
+	"go/token"
+	"go/types"
+	"log"
+	"os"
+
+	"golang.org/x/tools/go/gcexportdata"
+	"golang.org/x/tools/go/types/typeutil"
+)
+
+var packageFlag = flag.String("package", "", "alternative package to print")
+
+func main() {
+	log.SetPrefix("gcexportdata: ")
+	log.SetFlags(0)
+	flag.Usage = func() {
+		fmt.Fprintln(os.Stderr, "usage: gcexportdata [-package path] file.a")
+	}
+	flag.Parse()
+	if flag.NArg() != 1 {
+		flag.Usage()
+		os.Exit(2)
+	}
+	filename := flag.Args()[0]
+
+	f, err := os.Open(filename)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	r, err := gcexportdata.NewReader(f)
+	if err != nil {
+		log.Fatalf("%s: %s", filename, err)
+	}
+
+	// Decode the package.
+	const primary = "<primary>"
+	imports := make(map[string]*types.Package)
+	fset := token.NewFileSet()
+	pkg, err := gcexportdata.Read(r, fset, imports, primary)
+	if err != nil {
+		log.Fatalf("%s: %s", filename, err)
+	}
+
+	// Optionally select an indirectly mentioned package.
+	if *packageFlag != "" {
+		pkg = imports[*packageFlag]
+		if pkg == nil {
+			fmt.Fprintf(os.Stderr, "export data file %s does not mention %s; has:\n",
+				filename, *packageFlag)
+			for p := range imports {
+				if p != primary {
+					fmt.Fprintf(os.Stderr, "\t%s\n", p)
+				}
+			}
+			os.Exit(1)
+		}
+	}
+
+	// Print all package-level declarations, including non-exported ones.
+	fmt.Printf("package %s\n", pkg.Name())
+	for _, imp := range pkg.Imports() {
+		fmt.Printf("import %q\n", imp.Path())
+	}
+	qual := func(p *types.Package) string {
+		if pkg == p {
+			return ""
+		}
+		return p.Name()
+	}
+	scope := pkg.Scope()
+	for _, name := range scope.Names() {
+		obj := scope.Lookup(name)
+		fmt.Printf("%s: %s\n",
+			fset.Position(obj.Pos()),
+			types.ObjectString(obj, qual))
+
+		// For types, print each method.
+		if _, ok := obj.(*types.TypeName); ok {
+			for _, method := range typeutil.IntuitiveMethodSet(obj.Type(), nil) {
+				fmt.Printf("%s: %s\n",
+					fset.Position(method.Obj().Pos()),
+					types.SelectionString(method, qual))
+			}
+		}
+	}
+}

vendor/golang.org/x/tools/go/packages/external.go

@@ -18,7 +18,7 @@ import (
 
 // Driver
 type driverRequest struct {
-	Command    string            `json "command"`
+	Command    string            `json:"command"`
 	Mode       LoadMode          `json:"mode"`
 	Env        []string          `json:"env"`
 	BuildFlags []string          `json:"build_flags"`

vendor/golang.org/x/tools/go/packages/golist.go

@@ -78,7 +78,7 @@ func goListDriver(cfg *Config, patterns ...string) (*driverResponse, error) {
 	var sizes types.Sizes
 	var sizeserr error
 	var sizeswg sync.WaitGroup
-	if cfg.Mode&NeedTypesSizes != 0 {
+	if cfg.Mode&NeedTypesSizes != 0 || cfg.Mode&NeedTypes != 0 {
 		sizeswg.Add(1)
 		go func() {
 			sizes, sizeserr = getSizes(cfg)
@@ -128,7 +128,7 @@ extractQueries:
 	// patterns also requires a go list call, since it's the equivalent of
 	// ".".
 	if len(restPatterns) > 0 || len(patterns) == 0 {
-		dr, err := golistDriverCurrent(cfg, restPatterns...)
+		dr, err := golistDriver(cfg, restPatterns...)
 		if err != nil {
 			return nil, err
 		}
@@ -147,18 +147,18 @@ extractQueries:
 	var containsCandidates []string
 
 	if len(containFiles) != 0 {
-		if err := runContainsQueries(cfg, golistDriverCurrent, response, containFiles); err != nil {
+		if err := runContainsQueries(cfg, golistDriver, response, containFiles); err != nil {
 			return nil, err
 		}
 	}
 
 	if len(packagesNamed) != 0 {
-		if err := runNamedQueries(cfg, golistDriverCurrent, response, packagesNamed); err != nil {
+		if err := runNamedQueries(cfg, golistDriver, response, packagesNamed); err != nil {
 			return nil, err
 		}
 	}
 
-	modifiedPkgs, needPkgs, err := processGolistOverlay(cfg, response.dr)
+	modifiedPkgs, needPkgs, err := processGolistOverlay(cfg, response)
 	if err != nil {
 		return nil, err
 	}
@@ -166,17 +166,25 @@ extractQueries:
 		containsCandidates = append(containsCandidates, modifiedPkgs...)
 		containsCandidates = append(containsCandidates, needPkgs...)
 	}
-
-	if len(needPkgs) > 0 {
-		addNeededOverlayPackages(cfg, golistDriverCurrent, response, needPkgs)
-		if err != nil {
-			return nil, err
-		}
+	if err := addNeededOverlayPackages(cfg, golistDriver, response, needPkgs); err != nil {
+		return nil, err
 	}
 	// Check candidate packages for containFiles.
 	if len(containFiles) > 0 {
 		for _, id := range containsCandidates {
-			pkg := response.seenPackages[id]
+			pkg, ok := response.seenPackages[id]
+			if !ok {
+				response.addPackage(&Package{
+					ID: id,
+					Errors: []Error{
+						{
+							Kind: ListError,
+							Msg:  fmt.Sprintf("package %s expected but not seen", id),
+						},
+					},
+				})
+				continue
+			}
 			for _, f := range containFiles {
 				for _, g := range pkg.GoFiles {
 					if sameFile(f, g) {
@@ -191,6 +199,9 @@ extractQueries:
 }
 
 func addNeededOverlayPackages(cfg *Config, driver driver, response *responseDeduper, pkgs []string) error {
+	if len(pkgs) == 0 {
+		return nil
+	}
 	dr, err := driver(cfg, pkgs...)
 	if err != nil {
 		return err
@@ -198,6 +209,13 @@ func addNeededOverlayPackages(cfg *Config, driver driver, response *responseDedu
 	for _, pkg := range dr.Packages {
 		response.addPackage(pkg)
 	}
+	_, needPkgs, err := processGolistOverlay(cfg, response)
+	if err != nil {
+		return err
+	}
+	if err := addNeededOverlayPackages(cfg, driver, response, needPkgs); err != nil {
+		return err
+	}
 	return nil
 }
 
@@ -212,8 +230,16 @@ func runContainsQueries(cfg *Config, driver driver, response *responseDeduper, q
 			return fmt.Errorf("could not determine absolute path of file= query path %q: %v", query, err)
 		}
 		dirResponse, err := driver(cfg, pattern)
-		if err != nil {
-			return err
+		if err != nil || (len(dirResponse.Packages) == 1 && len(dirResponse.Packages[0].Errors) == 1) {
+			// There was an error loading the package. Try to load the file as an ad-hoc package.
+			// Usually the error will appear in a returned package, but may not if we're in modules mode
+			// and the ad-hoc is located outside a module.
+			var queryErr error
+			dirResponse, queryErr = driver(cfg, query)
+			if queryErr != nil {
+				// Return the original error if the attempt to fall back failed.
+				return err
+			}
 		}
 		isRoot := make(map[string]bool, len(dirResponse.Roots))
 		for _, root := range dirResponse.Roots {
@@ -540,10 +566,10 @@ func otherFiles(p *jsonPackage) [][]string {
 	return [][]string{p.CFiles, p.CXXFiles, p.MFiles, p.HFiles, p.FFiles, p.SFiles, p.SwigFiles, p.SwigCXXFiles, p.SysoFiles}
 }
 
-// golistDriverCurrent uses the "go list" command to expand the
-// pattern words and return metadata for the specified packages.
-// dir may be "" and env may be nil, as per os/exec.Command.
-func golistDriverCurrent(cfg *Config, words ...string) (*driverResponse, error) {
+// golistDriver uses the "go list" command to expand the pattern
+// words and return metadata for the specified packages. dir may be
+// "" and env may be nil, as per os/exec.Command.
+func golistDriver(cfg *Config, words ...string) (*driverResponse, error) {
 	// go list uses the following identifiers in ImportPath and Imports:
 	//
 	// 	"p"			-- importable package or main (command)
@@ -677,7 +703,7 @@ func golistDriverCurrent(cfg *Config, words ...string) (*driverResponse, error)
 		if p.Error != nil {
 			pkg.Errors = append(pkg.Errors, Error{
 				Pos: p.Error.Pos,
-				Msg: p.Error.Err,
+				Msg: strings.TrimSpace(p.Error.Err), // Trim to work around golang.org/issue/32363.
 			})
 		}
 
@@ -761,8 +787,31 @@ func invokeGo(cfg *Config, args ...string) (*bytes.Buffer, error) {
 		// the error in the Err section of stdout in case -e option is provided.
 		// This fix is provided for backwards compatibility.
 		if len(stderr.String()) > 0 && strings.Contains(stderr.String(), "named files must be .go files") {
-			output := fmt.Sprintf(`{"ImportPath": "","Incomplete": true,"Error": {"Pos": "","Err": %s}}`,
-				strconv.Quote(strings.Trim(stderr.String(), "\n")))
+			output := fmt.Sprintf(`{"ImportPath": "command-line-arguments","Incomplete": true,"Error": {"Pos": "","Err": %q}}`,
+				strings.Trim(stderr.String(), "\n"))
+			return bytes.NewBufferString(output), nil
+		}
+
+		// Workaround for #29280: go list -e has incorrect behavior when an ad-hoc package doesn't exist.
+		if len(stderr.String()) > 0 && strings.Contains(stderr.String(), "no such file or directory") {
+			output := fmt.Sprintf(`{"ImportPath": "command-line-arguments","Incomplete": true,"Error": {"Pos": "","Err": %q}}`,
+				strings.Trim(stderr.String(), "\n"))
+			return bytes.NewBufferString(output), nil
+		}
+
+		// Workaround for an instance of golang.org/issue/26755: go list -e  will return a non-zero exit
+		// status if there's a dependency on a package that doesn't exist. But it should return
+		// a zero exit status and set an error on that package.
+		if len(stderr.String()) > 0 && strings.Contains(stderr.String(), "no Go files in") {
+			// try to extract package name from string
+			stderrStr := stderr.String()
+			var importPath string
+			colon := strings.Index(stderrStr, ":")
+			if colon > 0 && strings.HasPrefix(stderrStr, "go build ") {
+				importPath = stderrStr[len("go build "):colon]
+			}
+			output := fmt.Sprintf(`{"ImportPath": %q,"Incomplete": true,"Error": {"Pos": "","Err": %q}}`,
+				importPath, strings.Trim(stderrStr, "\n"))
 			return bytes.NewBufferString(output), nil
 		}
 

vendor/golang.org/x/tools/go/packages/golist_overlay.go

@@ -1,83 +1,189 @@
 package packages
 
 import (
+	"bytes"
+	"encoding/json"
+	"fmt"
 	"go/parser"
 	"go/token"
+	"path"
 	"path/filepath"
 	"strconv"
 	"strings"
+	"sync"
 )
 
 // processGolistOverlay provides rudimentary support for adding
 // files that don't exist on disk to an overlay. The results can be
 // sometimes incorrect.
 // TODO(matloob): Handle unsupported cases, including the following:
-// - test files
-// - adding test and non-test files to test variants of packages
 // - determining the correct package to add given a new import path
-// - creating packages that don't exist
-func processGolistOverlay(cfg *Config, response *driverResponse) (modifiedPkgs, needPkgs []string, err error) {
+func processGolistOverlay(cfg *Config, response *responseDeduper) (modifiedPkgs, needPkgs []string, err error) {
 	havePkgs := make(map[string]string) // importPath -> non-test package ID
 	needPkgsSet := make(map[string]bool)
 	modifiedPkgsSet := make(map[string]bool)
 
-	for _, pkg := range response.Packages {
+	for _, pkg := range response.dr.Packages {
 		// This is an approximation of import path to id. This can be
 		// wrong for tests, vendored packages, and a number of other cases.
 		havePkgs[pkg.PkgPath] = pkg.ID
 	}
 
-outer:
-	for path, contents := range cfg.Overlay {
-		base := filepath.Base(path)
-		if strings.HasSuffix(path, "_test.go") {
-			// Overlays don't support adding new test files yet.
-			// TODO(matloob): support adding new test files.
+	var rootDirs map[string]string
+	var onceGetRootDirs sync.Once
+
+	// If no new imports are added, it is safe to avoid loading any needPkgs.
+	// Otherwise, it's hard to tell which package is actually being loaded
+	// (due to vendoring) and whether any modified package will show up
+	// in the transitive set of dependencies (because new imports are added,
+	// potentially modifying the transitive set of dependencies).
+	var overlayAddsImports bool
+
+	for opath, contents := range cfg.Overlay {
+		base := filepath.Base(opath)
+		dir := filepath.Dir(opath)
+		var pkg *Package
+		var testVariantOf *Package // if opath is a test file, this is the package it is testing
+		var fileExists bool
+		isTest := strings.HasSuffix(opath, "_test.go")
+		pkgName, ok := extractPackageName(opath, contents)
+		if !ok {
+			// Don't bother adding a file that doesn't even have a parsable package statement
+			// to the overlay.
 			continue
 		}
-		dir := filepath.Dir(path)
-		for _, pkg := range response.Packages {
-			var dirContains, fileExists bool
-			for _, f := range pkg.GoFiles {
-				if sameFile(filepath.Dir(f), dir) {
-					dirContains = true
+	nextPackage:
+		for _, p := range response.dr.Packages {
+			if pkgName != p.Name {
+				continue
+			}
+			for _, f := range p.GoFiles {
+				if !sameFile(filepath.Dir(f), dir) {
+					continue
 				}
+				if isTest && !hasTestFiles(p) {
+					// TODO(matloob): Are there packages other than the 'production' variant
+					// of a package that this can match? This shouldn't match the test main package
+					// because the file is generated in another directory.
+					testVariantOf = p
+					continue nextPackage
+				}
+				pkg = p
 				if filepath.Base(f) == base {
 					fileExists = true
 				}
 			}
-			if dirContains {
-				if !fileExists {
-					pkg.GoFiles = append(pkg.GoFiles, path) // TODO(matloob): should the file just be added to GoFiles?
-					pkg.CompiledGoFiles = append(pkg.CompiledGoFiles, path)
-					modifiedPkgsSet[pkg.ID] = true
-				}
-				imports, err := extractImports(path, contents)
+		}
+		// The overlay could have included an entirely new package.
+		if pkg == nil {
+			onceGetRootDirs.Do(func() {
+				rootDirs = determineRootDirs(cfg)
+			})
+			// Try to find the module or gopath dir the file is contained in.
+			// Then for modules, add the module opath to the beginning.
+			var pkgPath string
+			for rdir, rpath := range rootDirs {
+				// TODO(matloob): This doesn't properly handle symlinks.
+				r, err := filepath.Rel(rdir, dir)
 				if err != nil {
-					// Let the parser or type checker report errors later.
-					continue outer
+					continue
 				}
-				for _, imp := range imports {
-					_, found := pkg.Imports[imp]
-					if !found {
-						needPkgsSet[imp] = true
-						// TODO(matloob): Handle cases when the following block isn't correct.
-						// These include imports of test variants, imports of vendored packages, etc.
-						id, ok := havePkgs[imp]
-						if !ok {
-							id = imp
-						}
-						pkg.Imports[imp] = &Package{ID: id}
-					}
+				pkgPath = filepath.ToSlash(r)
+				if rpath != "" {
+					pkgPath = path.Join(rpath, pkgPath)
 				}
-				continue outer
+				// We only create one new package even it can belong in multiple modules or GOPATH entries.
+				// This is okay because tools (such as the LSP) that use overlays will recompute the overlay
+				// once the file is saved, and golist will do the right thing.
+				// TODO(matloob): Implement module tiebreaking?
+				break
+			}
+			if pkgPath == "" {
+				continue
+			}
+			isXTest := strings.HasSuffix(pkgName, "_test")
+			if isXTest {
+				pkgPath += "_test"
+			}
+			id := pkgPath
+			if isTest && !isXTest {
+				id = fmt.Sprintf("%s [%s.test]", pkgPath, pkgPath)
+			}
+			// Try to reclaim a package with the same id if it exists in the response.
+			for _, p := range response.dr.Packages {
+				if reclaimPackage(p, id, opath, contents) {
+					pkg = p
+					break
+				}
+			}
+			// Otherwise, create a new package
+			if pkg == nil {
+				pkg = &Package{PkgPath: pkgPath, ID: id, Name: pkgName, Imports: make(map[string]*Package)}
+				response.addPackage(pkg)
+				havePkgs[pkg.PkgPath] = id
+				// Add the production package's sources for a test variant.
+				if isTest && !isXTest && testVariantOf != nil {
+					pkg.GoFiles = append(pkg.GoFiles, testVariantOf.GoFiles...)
+					pkg.CompiledGoFiles = append(pkg.CompiledGoFiles, testVariantOf.CompiledGoFiles...)
+				}
+			}
+		}
+		if !fileExists {
+			pkg.GoFiles = append(pkg.GoFiles, opath)
+			// TODO(matloob): Adding the file to CompiledGoFiles can exhibit the wrong behavior
+			// if the file will be ignored due to its build tags.
+			pkg.CompiledGoFiles = append(pkg.CompiledGoFiles, opath)
+			modifiedPkgsSet[pkg.ID] = true
+		}
+		imports, err := extractImports(opath, contents)
+		if err != nil {
+			// Let the parser or type checker report errors later.
+			continue
+		}
+		for _, imp := range imports {
+			_, found := pkg.Imports[imp]
+			if !found {
+				overlayAddsImports = true
+				// TODO(matloob): Handle cases when the following block isn't correct.
+				// These include imports of test variants, imports of vendored packages, etc.
+				id, ok := havePkgs[imp]
+				if !ok {
+					id = imp
+				}
+				pkg.Imports[imp] = &Package{ID: id}
+			}
+		}
+		continue
+	}
+
+	// toPkgPath tries to guess the package path given the id.
+	// This isn't always correct -- it's certainly wrong for
+	// vendored packages' paths.
+	toPkgPath := func(id string) string {
+		// TODO(matloob): Handle vendor paths.
+		i := strings.IndexByte(id, ' ')
+		if i >= 0 {
+			return id[:i]
+		}
+		return id
+	}
+
+	// Do another pass now that new packages have been created to determine the
+	// set of missing packages.
+	for _, pkg := range response.dr.Packages {
+		for _, imp := range pkg.Imports {
+			pkgPath := toPkgPath(imp.ID)
+			if _, ok := havePkgs[pkgPath]; !ok {
+				needPkgsSet[pkgPath] = true
 			}
 		}
 	}
 
-	needPkgs = make([]string, 0, len(needPkgsSet))
-	for pkg := range needPkgsSet {
-		needPkgs = append(needPkgs, pkg)
+	if overlayAddsImports {
+		needPkgs = make([]string, 0, len(needPkgsSet))
+		for pkg := range needPkgsSet {
+			needPkgs = append(needPkgs, pkg)
+		}
 	}
 	modifiedPkgs = make([]string, 0, len(modifiedPkgsSet))
 	for pkg := range modifiedPkgsSet {
@@ -86,6 +192,55 @@ outer:
 	return modifiedPkgs, needPkgs, err
 }
 
+func hasTestFiles(p *Package) bool {
+	for _, f := range p.GoFiles {
+		if strings.HasSuffix(f, "_test.go") {
+			return true
+		}
+	}
+	return false
+}
+
+// determineRootDirs returns a mapping from directories code can be contained in to the
+// corresponding import path prefixes of those directories.
+// Its result is used to try to determine the import path for a package containing
+// an overlay file.
+func determineRootDirs(cfg *Config) map[string]string {
+	// Assume modules first:
+	out, err := invokeGo(cfg, "list", "-m", "-json", "all")
+	if err != nil {
+		return determineRootDirsGOPATH(cfg)
+	}
+	m := map[string]string{}
+	type jsonMod struct{ Path, Dir string }
+	for dec := json.NewDecoder(out); dec.More(); {
+		mod := new(jsonMod)
+		if err := dec.Decode(mod); err != nil {
+			return m // Give up and return an empty map. Package won't be found for overlay.
+		}
+		if mod.Dir != "" && mod.Path != "" {
+			// This is a valid module; add it to the map.
+			m[mod.Dir] = mod.Path
+		}
+	}
+	return m
+}
+
+func determineRootDirsGOPATH(cfg *Config) map[string]string {
+	m := map[string]string{}
+	out, err := invokeGo(cfg, "env", "GOPATH")
+	if err != nil {
+		// Could not determine root dir mapping. Everything is best-effort, so just return an empty map.
+		// When we try to find the import path for a directory, there will be no root-dir match and
+		// we'll give up.
+		return m
+	}
+	for _, p := range filepath.SplitList(string(bytes.TrimSpace(out.Bytes()))) {
+		m[filepath.Join(p, "src")] = ""
+	}
+	return m
+}
+
 func extractImports(filename string, contents []byte) ([]string, error) {
 	f, err := parser.ParseFile(token.NewFileSet(), filename, contents, parser.ImportsOnly) // TODO(matloob): reuse fileset?
 	if err != nil {
@@ -102,3 +257,44 @@ func extractImports(filename string, contents []byte) ([]string, error) {
 	}
 	return res, nil
 }
+
+// reclaimPackage attempts to reuse a package that failed to load in an overlay.
+//
+// If the package has errors and has no Name, GoFiles, or Imports,
+// then it's possible that it doesn't yet exist on disk.
+func reclaimPackage(pkg *Package, id string, filename string, contents []byte) bool {
+	// TODO(rstambler): Check the message of the actual error?
+	// It differs between $GOPATH and module mode.
+	if pkg.ID != id {
+		return false
+	}
+	if len(pkg.Errors) != 1 {
+		return false
+	}
+	if pkg.Name != "" || pkg.ExportFile != "" {
+		return false
+	}
+	if len(pkg.GoFiles) > 0 || len(pkg.CompiledGoFiles) > 0 || len(pkg.OtherFiles) > 0 {
+		return false
+	}
+	if len(pkg.Imports) > 0 {
+		return false
+	}
+	pkgName, ok := extractPackageName(filename, contents)
+	if !ok {
+		return false
+	}
+	pkg.Name = pkgName
+	pkg.Errors = nil
+	return true
+}
+
+func extractPackageName(filename string, contents []byte) (string, bool) {
+	// TODO(rstambler): Check the message of the actual error?
+	// It differs between $GOPATH and module mode.
+	f, err := parser.ParseFile(token.NewFileSet(), filename, contents, parser.PackageClauseOnly) // TODO(matloob): reuse fileset?
+	if err != nil {
+		return "", false
+	}
+	return f.Name.Name, true
+}

vendor/golang.org/x/tools/go/packages/packages.go

@@ -25,24 +25,16 @@ import (
 	"golang.org/x/tools/go/gcexportdata"
 )
 
-// A LoadMode specifies the amount of detail to return when loading.
-// Higher-numbered modes cause Load to return more information,
-// but may be slower. Load may return more information than requested.
+// A LoadMode controls the amount of detail to return when loading.
+// The bits below can be combined to specify which fields should be
+// filled in the result packages.
+// The zero value is a special case, equivalent to combining
+// the NeedName, NeedFiles, and NeedCompiledGoFiles bits.
+// ID and Errors (if present) will always be filled.
+// Load may return more information than requested.
 type LoadMode int
 
 const (
-	// The following constants are used to specify which fields of the Package
-	// should be filled when loading is done. As a special case to provide
-	// backwards compatibility, a LoadMode of 0 is equivalent to LoadFiles.
-	// For all other LoadModes, the bits below specify which fields will be filled
-	// in the result packages.
-	// WARNING: This part of the go/packages API is EXPERIMENTAL. It might
-	// be changed or removed up until April 15 2019. After that date it will
-	// be frozen.
-	// TODO(matloob): Remove this comment on April 15.
-
-	// ID and Errors (if present) will always be filled.
-
 	// NeedName adds Name and PkgPath.
 	NeedName LoadMode = 1 << iota
 
@@ -77,30 +69,24 @@ const (
 )
 
 const (
-	// LoadFiles finds the packages and computes their source file lists.
-	// Package fields: ID, Name, Errors, GoFiles, CompiledGoFiles, and OtherFiles.
+	// Deprecated: LoadFiles exists for historical compatibility
+	// and should not be used. Please directly specify the needed fields using the Need values.
 	LoadFiles = NeedName | NeedFiles | NeedCompiledGoFiles
 
-	// LoadImports adds import information for each package
-	// and its dependencies.
-	// Package fields added: Imports.
+	// Deprecated: LoadImports exists for historical compatibility
+	// and should not be used. Please directly specify the needed fields using the Need values.
 	LoadImports = LoadFiles | NeedImports | NeedDeps
 
-	// LoadTypes adds type information for package-level
-	// declarations in the packages matching the patterns.
-	// Package fields added: Types, Fset, and IllTyped.
-	// This mode uses type information provided by the build system when
-	// possible, and may fill in the ExportFile field.
-	LoadTypes = LoadImports | NeedTypes
+	// Deprecated: LoadTypes exists for historical compatibility
+	// and should not be used. Please directly specify the needed fields using the Need values.
+	LoadTypes = LoadImports | NeedTypes | NeedTypesSizes
 
-	// LoadSyntax adds typed syntax trees for the packages matching the patterns.
-	// Package fields added: Syntax, and TypesInfo, for direct pattern matches only.
-	LoadSyntax = LoadTypes | NeedSyntax | NeedTypesInfo | NeedTypesSizes
+	// Deprecated: LoadSyntax exists for historical compatibility
+	// and should not be used. Please directly specify the needed fields using the Need values.
+	LoadSyntax = LoadTypes | NeedSyntax | NeedTypesInfo
 
-	// LoadAllSyntax adds typed syntax trees for the packages matching the patterns
-	// and all dependencies.
-	// Package fields added: Types, Fset, IllTyped, Syntax, and TypesInfo,
-	// for all packages in the import graph.
+	// Deprecated: LoadAllSyntax exists for historical compatibility
+	// and should not be used. Please directly specify the needed fields using the Need values.
 	LoadAllSyntax = LoadSyntax
 )
 
@@ -137,7 +123,7 @@ type Config struct {
 	BuildFlags []string
 
 	// Fset provides source position information for syntax trees and types.
-	// If Fset is nil, the loader will create a new FileSet.
+	// If Fset is nil, Load will use a new fileset, but preserve Fset's value.
 	Fset *token.FileSet
 
 	// ParseFile is called to read and parse each file
@@ -275,9 +261,9 @@ type Package struct {
 	Imports map[string]*Package
 
 	// Types provides type information for the package.
-	// Modes LoadTypes and above set this field for packages matching the
-	// patterns; type information for dependencies may be missing or incomplete.
-	// Mode LoadAllSyntax sets this field for all packages, including dependencies.
+	// The NeedTypes LoadMode bit sets this field for packages matching the
+	// patterns; type information for dependencies may be missing or incomplete,
+	// unless NeedDeps and NeedImports are also set.
 	Types *types.Package
 
 	// Fset provides position information for Types, TypesInfo, and Syntax.
@@ -290,8 +276,9 @@ type Package struct {
 
 	// Syntax is the package's syntax trees, for the files listed in CompiledGoFiles.
 	//
-	// Mode LoadSyntax sets this field for packages matching the patterns.
-	// Mode LoadAllSyntax sets this field for all packages, including dependencies.
+	// The NeedSyntax LoadMode bit populates this field for packages matching the patterns.
+	// If NeedDeps and NeedImports are also set, this field will also be populated
+	// for dependencies.
 	Syntax []*ast.File
 
 	// TypesInfo provides type information about the package's syntax trees.
@@ -418,17 +405,33 @@ type loaderPackage struct {
 type loader struct {
 	pkgs map[string]*loaderPackage
 	Config
-	sizes    types.Sizes
-	exportMu sync.Mutex // enforces mutual exclusion of exportdata operations
+	sizes        types.Sizes
+	parseCache   map[string]*parseValue
+	parseCacheMu sync.Mutex
+	exportMu     sync.Mutex // enforces mutual exclusion of exportdata operations
+
+	// TODO(matloob): Add an implied mode here and use that instead of mode.
+	// Implied mode would contain all the fields we need the data for so we can
+	// get the actually requested fields. We'll zero them out before returning
+	// packages to the user. This will make it easier for us to get the conditions
+	// where we need certain modes right.
+}
+
+type parseValue struct {
+	f     *ast.File
+	err   error
+	ready chan struct{}
 }
 
 func newLoader(cfg *Config) *loader {
-	ld := &loader{}
+	ld := &loader{
+		parseCache: map[string]*parseValue{},
+	}
 	if cfg != nil {
 		ld.Config = *cfg
 	}
 	if ld.Config.Mode == 0 {
-		ld.Config.Mode = LoadFiles // Preserve zero behavior of Mode for backwards compatibility.
+		ld.Config.Mode = NeedName | NeedFiles | NeedCompiledGoFiles // Preserve zero behavior of Mode for backwards compatibility.
 	}
 	if ld.Config.Env == nil {
 		ld.Config.Env = os.Environ()
@@ -451,12 +454,8 @@ func newLoader(cfg *Config) *loader {
 		// because we load source if export data is missing.
 		if ld.ParseFile == nil {
 			ld.ParseFile = func(fset *token.FileSet, filename string, src []byte) (*ast.File, error) {
-				var isrc interface{}
-				if src != nil {
-					isrc = src
-				}
 				const mode = parser.AllErrors | parser.ParseComments
-				return parser.ParseFile(fset, filename, isrc, mode)
+				return parser.ParseFile(fset, filename, src, mode)
 			}
 		}
 	}
@@ -528,39 +527,46 @@ func (ld *loader) refine(roots []string, list ...*Package) ([]*Package, error) {
 		lpkg.color = grey
 		stack = append(stack, lpkg) // push
 		stubs := lpkg.Imports       // the structure form has only stubs with the ID in the Imports
-		lpkg.Imports = make(map[string]*Package, len(stubs))
-		for importPath, ipkg := range stubs {
-			var importErr error
-			imp := ld.pkgs[ipkg.ID]
-			if imp == nil {
-				// (includes package "C" when DisableCgo)
-				importErr = fmt.Errorf("missing package: %q", ipkg.ID)
-			} else if imp.color == grey {
-				importErr = fmt.Errorf("import cycle: %s", stack)
-			}
-			if importErr != nil {
-				if lpkg.importErrors == nil {
-					lpkg.importErrors = make(map[string]error)
+		// If NeedImports isn't set, the imports fields will all be zeroed out.
+		// If NeedDeps isn't also set we want to keep the stubs.
+		if ld.Mode&NeedImports != 0 && ld.Mode&NeedDeps != 0 {
+			lpkg.Imports = make(map[string]*Package, len(stubs))
+			for importPath, ipkg := range stubs {
+				var importErr error
+				imp := ld.pkgs[ipkg.ID]
+				if imp == nil {
+					// (includes package "C" when DisableCgo)
+					importErr = fmt.Errorf("missing package: %q", ipkg.ID)
+				} else if imp.color == grey {
+					importErr = fmt.Errorf("import cycle: %s", stack)
+				}
+				if importErr != nil {
+					if lpkg.importErrors == nil {
+						lpkg.importErrors = make(map[string]error)
+					}
+					lpkg.importErrors[importPath] = importErr
+					continue
 				}
-				lpkg.importErrors[importPath] = importErr
-				continue
-			}
 
-			if visit(imp) {
-				lpkg.needsrc = true
+				if visit(imp) {
+					lpkg.needsrc = true
+				}
+				lpkg.Imports[importPath] = imp.Package
 			}
-			lpkg.Imports[importPath] = imp.Package
 		}
 		if lpkg.needsrc {
 			srcPkgs = append(srcPkgs, lpkg)
 		}
+		if ld.Mode&NeedTypesSizes != 0 {
+			lpkg.TypesSizes = ld.sizes
+		}
 		stack = stack[:len(stack)-1] // pop
 		lpkg.color = black
 
 		return lpkg.needsrc
 	}
 
-	if ld.Mode&NeedImports == 0 {
+	if ld.Mode&(NeedImports|NeedDeps) == 0 {
 		// We do this to drop the stub import packages that we are not even going to try to resolve.
 		for _, lpkg := range initial {
 			lpkg.Imports = nil
@@ -571,7 +577,7 @@ func (ld *loader) refine(roots []string, list ...*Package) ([]*Package, error) {
 			visit(lpkg)
 		}
 	}
-	if ld.Mode&NeedDeps != 0 {
+	if ld.Mode&NeedDeps != 0 { // TODO(matloob): This is only the case if NeedTypes is also set, right?
 		for _, lpkg := range srcPkgs {
 			// Complete type information is required for the
 			// immediate dependencies of each source package.
@@ -599,46 +605,48 @@ func (ld *loader) refine(roots []string, list ...*Package) ([]*Package, error) {
 	importPlaceholders := make(map[string]*Package)
 	for i, lpkg := range initial {
 		result[i] = lpkg.Package
+	}
+	for i := range ld.pkgs {
 		// Clear all unrequested fields, for extra de-Hyrum-ization.
 		if ld.Mode&NeedName == 0 {
-			result[i].Name = ""
-			result[i].PkgPath = ""
+			ld.pkgs[i].Name = ""
+			ld.pkgs[i].PkgPath = ""
 		}
 		if ld.Mode&NeedFiles == 0 {
-			result[i].GoFiles = nil
-			result[i].OtherFiles = nil
+			ld.pkgs[i].GoFiles = nil
+			ld.pkgs[i].OtherFiles = nil
 		}
 		if ld.Mode&NeedCompiledGoFiles == 0 {
-			result[i].CompiledGoFiles = nil
+			ld.pkgs[i].CompiledGoFiles = nil
 		}
 		if ld.Mode&NeedImports == 0 {
-			result[i].Imports = nil
+			ld.pkgs[i].Imports = nil
 		}
 		if ld.Mode&NeedExportsFile == 0 {
-			result[i].ExportFile = ""
+			ld.pkgs[i].ExportFile = ""
 		}
 		if ld.Mode&NeedTypes == 0 {
-			result[i].Types = nil
-			result[i].Fset = nil
-			result[i].IllTyped = false
+			ld.pkgs[i].Types = nil
+			ld.pkgs[i].Fset = nil
+			ld.pkgs[i].IllTyped = false
 		}
 		if ld.Mode&NeedSyntax == 0 {
-			result[i].Syntax = nil
+			ld.pkgs[i].Syntax = nil
 		}
 		if ld.Mode&NeedTypesInfo == 0 {
-			result[i].TypesInfo = nil
+			ld.pkgs[i].TypesInfo = nil
 		}
 		if ld.Mode&NeedTypesSizes == 0 {
-			result[i].TypesSizes = nil
+			ld.pkgs[i].TypesSizes = nil
 		}
 		if ld.Mode&NeedDeps == 0 {
-			for j, pkg := range result[i].Imports {
+			for j, pkg := range ld.pkgs[i].Imports {
 				ph, ok := importPlaceholders[pkg.ID]
 				if !ok {
 					ph = &Package{ID: pkg.ID}
 					importPlaceholders[pkg.ID] = ph
 				}
-				result[i].Imports[j] = ph
+				ld.pkgs[i].Imports[j] = ph
 			}
 		}
 	}
@@ -670,7 +678,7 @@ func (ld *loader) loadRecursive(lpkg *loaderPackage) {
 // loadPackage loads the specified package.
 // It must be called only once per Package,
 // after immediate dependencies are loaded.
-// Precondition: ld.Mode >= LoadTypes.
+// Precondition: ld.Mode & NeedTypes.
 func (ld *loader) loadPackage(lpkg *loaderPackage) {
 	if lpkg.PkgPath == "unsafe" {
 		// Fill in the blanks to avoid surprises.
@@ -853,6 +861,42 @@ func (f importerFunc) Import(path string) (*types.Package, error) { return f(pat
 // the number of parallel I/O calls per process.
 var ioLimit = make(chan bool, 20)
 
+func (ld *loader) parseFile(filename string) (*ast.File, error) {
+	ld.parseCacheMu.Lock()
+	v, ok := ld.parseCache[filename]
+	if ok {
+		// cache hit
+		ld.parseCacheMu.Unlock()
+		<-v.ready
+	} else {
+		// cache miss
+		v = &parseValue{ready: make(chan struct{})}
+		ld.parseCache[filename] = v
+		ld.parseCacheMu.Unlock()
+
+		var src []byte
+		for f, contents := range ld.Config.Overlay {
+			if sameFile(f, filename) {
+				src = contents
+			}
+		}
+		var err error
+		if src == nil {
+			ioLimit <- true // wait
+			src, err = ioutil.ReadFile(filename)
+			<-ioLimit // signal
+		}
+		if err != nil {
+			v.err = err
+		} else {
+			v.f, v.err = ld.ParseFile(ld.Fset, filename, src)
+		}
+
+		close(v.ready)
+	}
+	return v.f, v.err
+}
+
 // parseFiles reads and parses the Go source files and returns the ASTs
 // of the ones that could be at least partially parsed, along with a
 // list of I/O and parse errors encountered.
@@ -873,24 +917,7 @@ func (ld *loader) parseFiles(filenames []string) ([]*ast.File, []error) {
 		}
 		wg.Add(1)
 		go func(i int, filename string) {
-			ioLimit <- true // wait
-			// ParseFile may return both an AST and an error.
-			var src []byte
-			for f, contents := range ld.Config.Overlay {
-				if sameFile(f, filename) {
-					src = contents
-				}
-			}
-			var err error
-			if src == nil {
-				src, err = ioutil.ReadFile(filename)
-			}
-			if err != nil {
-				parsed[i], errors[i] = nil, err
-			} else {
-				parsed[i], errors[i] = ld.ParseFile(ld.Fset, filename, src)
-			}
-			<-ioLimit // signal
+			parsed[i], errors[i] = ld.parseFile(filename)
 			wg.Done()
 		}(i, file)
 	}