Update vendored dependencies

vendor/golang.org/x/tools/go/packages/external.go

@@ -12,6 +12,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"os"
 	"os/exec"
 	"strings"
 )
@@ -76,12 +77,17 @@ func findExternalDriver(cfg *Config) driver {
 		}
 
 		buf := new(bytes.Buffer)
+		stderr := new(bytes.Buffer)
 		cmd := exec.CommandContext(cfg.Context, tool, words...)
 		cmd.Dir = cfg.Dir
 		cmd.Env = cfg.Env
 		cmd.Stdin = bytes.NewReader(req)
 		cmd.Stdout = buf
-		cmd.Stderr = new(bytes.Buffer)
+		cmd.Stderr = stderr
+		if len(stderr.Bytes()) != 0 && os.Getenv("GOPACKAGESPRINTDRIVERERRORS") != "" {
+			fmt.Fprintf(os.Stderr, "%s stderr: <<%s>>\n", cmdDebugStr(cmd, words...), stderr)
+		}
+
 		if err := cmd.Run(); err != nil {
 			return nil, fmt.Errorf("%v: %v: %s", tool, err, cmd.Stderr)
 		}

vendor/golang.org/x/tools/go/packages/golist.go

@@ -26,6 +26,7 @@ import (
 	"golang.org/x/tools/go/internal/packagesdriver"
 	"golang.org/x/tools/internal/gopathwalk"
 	"golang.org/x/tools/internal/semver"
+	"golang.org/x/tools/internal/span"
 )
 
 // debug controls verbose logging.
@@ -109,6 +110,7 @@ func goListDriver(cfg *Config, patterns ...string) (*driverResponse, error) {
 			sizeswg.Done()
 		}()
 	}
+	defer sizeswg.Wait()
 
 	// start fetching rootDirs
 	var info goInfo
@@ -127,6 +129,10 @@ func goListDriver(cfg *Config, patterns ...string) (*driverResponse, error) {
 		return &info
 	}
 
+	// Ensure that we don't leak goroutines: Load is synchronous, so callers will
+	// not expect it to access the fields of cfg after the call returns.
+	defer getGoInfo()
+
 	// always pass getGoInfo to golistDriver
 	golistDriver := func(cfg *Config, patterns ...string) (*driverResponse, error) {
 		return golistDriver(cfg, getGoInfo, patterns...)
@@ -264,10 +270,7 @@ func addNeededOverlayPackages(cfg *Config, driver driver, response *responseDedu
 	if err != nil {
 		return err
 	}
-	if err := addNeededOverlayPackages(cfg, driver, response, needPkgs, getGoInfo); err != nil {
-		return err
-	}
-	return nil
+	return addNeededOverlayPackages(cfg, driver, response, needPkgs, getGoInfo)
 }
 
 func runContainsQueries(cfg *Config, driver driver, response *responseDeduper, queries []string, goInfo func() *goInfo) error {
@@ -281,30 +284,42 @@ func runContainsQueries(cfg *Config, driver driver, response *responseDeduper, q
 			return fmt.Errorf("could not determine absolute path of file= query path %q: %v", query, err)
 		}
 		dirResponse, err := driver(cfg, pattern)
-		if err != nil || (len(dirResponse.Packages) == 1 && len(dirResponse.Packages[0].Errors) == 1) {
-			// There was an error loading the package. Try to load the file as an ad-hoc package.
-			// Usually the error will appear in a returned package, but may not if we're in modules mode
-			// and the ad-hoc is located outside a module.
+		if err != nil {
 			var queryErr error
-			dirResponse, queryErr = driver(cfg, query)
-			if queryErr != nil {
-				// Return the original error if the attempt to fall back failed.
-				return err
+			if dirResponse, queryErr = adHocPackage(cfg, driver, pattern, query); queryErr != nil {
+				return err // return the original error
 			}
-			// Special case to handle issue #33482:
-			// If this is a file= query for ad-hoc packages where the file only exists on an overlay,
-			// and exists outside of a module, add the file in for the package.
-			if len(dirResponse.Packages) == 1 && len(dirResponse.Packages) == 1 &&
-				dirResponse.Packages[0].ID == "command-line-arguments" && len(dirResponse.Packages[0].GoFiles) == 0 {
-				filename := filepath.Join(pattern, filepath.Base(query)) // avoid recomputing abspath
-				// TODO(matloob): check if the file is outside of a root dir?
-				for path := range cfg.Overlay {
-					if path == filename {
-						dirResponse.Packages[0].Errors = nil
-						dirResponse.Packages[0].GoFiles = []string{path}
-						dirResponse.Packages[0].CompiledGoFiles = []string{path}
-					}
+		}
+		// `go list` can report errors for files that are not listed as part of a package's GoFiles.
+		// In the case of an invalid Go file, we should assume that it is part of package if only
+		// one package is in the response. The file may have valid contents in an overlay.
+		if len(dirResponse.Packages) == 1 {
+			pkg := dirResponse.Packages[0]
+			for i, err := range pkg.Errors {
+				s := errorSpan(err)
+				if !s.IsValid() {
+					break
 				}
+				if len(pkg.CompiledGoFiles) == 0 {
+					break
+				}
+				dir := filepath.Dir(pkg.CompiledGoFiles[0])
+				filename := filepath.Join(dir, filepath.Base(s.URI().Filename()))
+				if info, err := os.Stat(filename); err != nil || info.IsDir() {
+					break
+				}
+				if !contains(pkg.CompiledGoFiles, filename) {
+					pkg.CompiledGoFiles = append(pkg.CompiledGoFiles, filename)
+					pkg.GoFiles = append(pkg.GoFiles, filename)
+					pkg.Errors = append(pkg.Errors[:i], pkg.Errors[i+1:]...)
+				}
+			}
+		}
+		// A final attempt to construct an ad-hoc package.
+		if len(dirResponse.Packages) == 1 && len(dirResponse.Packages[0].Errors) == 1 {
+			var queryErr error
+			if dirResponse, queryErr = adHocPackage(cfg, driver, pattern, query); queryErr != nil {
+				return err // return the original error
 			}
 		}
 		isRoot := make(map[string]bool, len(dirResponse.Roots))
@@ -332,6 +347,75 @@ func runContainsQueries(cfg *Config, driver driver, response *responseDeduper, q
 	return nil
 }
 
+// adHocPackage attempts to construct an ad-hoc package given a query that failed.
+func adHocPackage(cfg *Config, driver driver, pattern, query string) (*driverResponse, error) {
+	// There was an error loading the package. Try to load the file as an ad-hoc package.
+	// Usually the error will appear in a returned package, but may not if we're in modules mode
+	// and the ad-hoc is located outside a module.
+	dirResponse, err := driver(cfg, query)
+	if err != nil {
+		return nil, err
+	}
+	// If we get nothing back from `go list`, try to make this file into its own ad-hoc package.
+	if len(dirResponse.Packages) == 0 && err == nil {
+		dirResponse.Packages = append(dirResponse.Packages, &Package{
+			ID:              "command-line-arguments",
+			PkgPath:         query,
+			GoFiles:         []string{query},
+			CompiledGoFiles: []string{query},
+			Imports:         make(map[string]*Package),
+		})
+		dirResponse.Roots = append(dirResponse.Roots, "command-line-arguments")
+	}
+	// Special case to handle issue #33482:
+	// If this is a file= query for ad-hoc packages where the file only exists on an overlay,
+	// and exists outside of a module, add the file in for the package.
+	if len(dirResponse.Packages) == 1 && (dirResponse.Packages[0].ID == "command-line-arguments" ||
+		filepath.ToSlash(dirResponse.Packages[0].PkgPath) == filepath.ToSlash(query)) {
+		if len(dirResponse.Packages[0].GoFiles) == 0 {
+			filename := filepath.Join(pattern, filepath.Base(query)) // avoid recomputing abspath
+			// TODO(matloob): check if the file is outside of a root dir?
+			for path := range cfg.Overlay {
+				if path == filename {
+					dirResponse.Packages[0].Errors = nil
+					dirResponse.Packages[0].GoFiles = []string{path}
+					dirResponse.Packages[0].CompiledGoFiles = []string{path}
+				}
+			}
+		}
+	}
+	return dirResponse, nil
+}
+
+func contains(files []string, filename string) bool {
+	for _, f := range files {
+		if f == filename {
+			return true
+		}
+	}
+	return false
+}
+
+// errorSpan attempts to parse a standard `go list` error message
+// by stripping off the trailing error message.
+//
+// It works only on errors whose message is prefixed by colon,
+// followed by a space (": "). For example:
+//
+//   attributes.go:13:1: expected 'package', found 'type'
+//
+func errorSpan(err Error) span.Span {
+	if err.Pos == "" {
+		input := strings.TrimSpace(err.Msg)
+		msgIndex := strings.Index(input, ": ")
+		if msgIndex < 0 {
+			return span.Parse(input)
+		}
+		return span.Parse(input[:msgIndex])
+	}
+	return span.Parse(err.Pos)
+}
+
 // modCacheRegexp splits a path in a module cache into module, module version, and package.
 var modCacheRegexp = regexp.MustCompile(`(.*)@([^/\\]*)(.*)`)
 
@@ -395,6 +479,10 @@ func runNamedQueries(cfg *Config, driver driver, response *responseDeduper, quer
 		}
 
 		files, err := ioutil.ReadDir(modRoot)
+		if err != nil {
+			panic(err) // See above.
+		}
+
 		for _, f := range files {
 			if strings.HasSuffix(f.Name(), ".go") {
 				simpleMatches = append(simpleMatches, rel)
@@ -462,7 +550,7 @@ func runNamedQueries(cfg *Config, driver driver, response *responseDeduper, quer
 		// We're only trying to look at stuff in the module cache, so
 		// disable the network. This should speed things up, and has
 		// prevented errors in at least one case, #28518.
-		tmpCfg.Env = append(append([]string{"GOPROXY=off"}, cfg.Env...))
+		tmpCfg.Env = append([]string{"GOPROXY=off"}, cfg.Env...)
 
 		var err error
 		tmpCfg.Dir, err = ioutil.TempDir("", "gopackages-modquery")
@@ -510,17 +598,29 @@ func roots(cfg *Config) ([]gopathwalk.Root, string, error) {
 
 	var roots []gopathwalk.Root
 	// Always add GOROOT.
-	roots = append(roots, gopathwalk.Root{filepath.Join(goroot, "/src"), gopathwalk.RootGOROOT})
+	roots = append(roots, gopathwalk.Root{
+		Path: filepath.Join(goroot, "/src"),
+		Type: gopathwalk.RootGOROOT,
+	})
 	// If modules are enabled, scan the module dir.
 	if modDir != "" {
-		roots = append(roots, gopathwalk.Root{modDir, gopathwalk.RootCurrentModule})
+		roots = append(roots, gopathwalk.Root{
+			Path: modDir,
+			Type: gopathwalk.RootCurrentModule,
+		})
 	}
 	// Add either GOPATH/src or GOPATH/pkg/mod, depending on module mode.
 	for _, p := range gopath {
 		if modDir != "" {
-			roots = append(roots, gopathwalk.Root{filepath.Join(p, "/pkg/mod"), gopathwalk.RootModuleCache})
+			roots = append(roots, gopathwalk.Root{
+				Path: filepath.Join(p, "/pkg/mod"),
+				Type: gopathwalk.RootModuleCache,
+			})
 		} else {
-			roots = append(roots, gopathwalk.Root{filepath.Join(p, "/src"), gopathwalk.RootGOPATH})
+			roots = append(roots, gopathwalk.Root{
+				Path: filepath.Join(p, "/src"),
+				Type: gopathwalk.RootGOPATH,
+			})
 		}
 	}
 
@@ -637,9 +737,9 @@ func golistDriver(cfg *Config, rootsDirs func() *goInfo, words ...string) (*driv
 	// go list uses the following identifiers in ImportPath and Imports:
 	//
 	// 	"p"			-- importable package or main (command)
-	//      "q.test"		-- q's test executable
+	// 	"q.test"		-- q's test executable
 	// 	"p [q.test]"		-- variant of p as built for q's test executable
-	//	"q_test [q.test]"	-- q's external test package
+	// 	"q_test [q.test]"	-- q's external test package
 	//
 	// The packages p that are built differently for a test q.test
 	// are q itself, plus any helpers used by the external test q_test,
@@ -682,7 +782,7 @@ func golistDriver(cfg *Config, rootsDirs func() *goInfo, words ...string) (*driv
 		// contained in a known module or GOPATH entry. This will allow the package to be
 		// properly "reclaimed" when overlays are processed.
 		if filepath.IsAbs(p.ImportPath) && p.Error != nil {
-			pkgPath, ok := getPkgPath(p.ImportPath, rootsDirs)
+			pkgPath, ok := getPkgPath(cfg, p.ImportPath, rootsDirs)
 			if ok {
 				p.ImportPath = pkgPath
 			}
@@ -792,15 +892,31 @@ func golistDriver(cfg *Config, rootsDirs func() *goInfo, words ...string) (*driv
 }
 
 // getPkgPath finds the package path of a directory if it's relative to a root directory.
-func getPkgPath(dir string, goInfo func() *goInfo) (string, bool) {
+func getPkgPath(cfg *Config, dir string, goInfo func() *goInfo) (string, bool) {
+	absDir, err := filepath.Abs(dir)
+	if err != nil {
+		cfg.Logf("error getting absolute path of %s: %v", dir, err)
+		return "", false
+	}
 	for rdir, rpath := range goInfo().rootDirs {
+		absRdir, err := filepath.Abs(rdir)
+		if err != nil {
+			cfg.Logf("error getting absolute path of %s: %v", rdir, err)
+			continue
+		}
+		// Make sure that the directory is in the module,
+		// to avoid creating a path relative to another module.
+		if !strings.HasPrefix(absDir, absRdir) {
+			cfg.Logf("%s does not have prefix %s", absDir, absRdir)
+			continue
+		}
 		// TODO(matloob): This doesn't properly handle symlinks.
 		r, err := filepath.Rel(rdir, dir)
 		if err != nil {
 			continue
 		}
 		if rpath != "" {
-			// We choose only ore root even though the directory even it can belong in multiple modules
+			// We choose only one root even though the directory even it can belong in multiple modules
 			// or GOPATH entries. This is okay because we only need to work with absolute dirs when a
 			// file is missing from disk, for instance when gopls calls go/packages in an overlay.
 			// Once the file is saved, gopls, or the next invocation of the tool will get the correct
@@ -808,6 +924,7 @@ func getPkgPath(dir string, goInfo func() *goInfo) (string, bool) {
 			// TODO(matloob): Implement module tiebreaking?
 			return path.Join(rpath, filepath.ToSlash(r)), true
 		}
+		return filepath.ToSlash(r), true
 	}
 	return "", false
 }
@@ -859,7 +976,7 @@ func invokeGo(cfg *Config, args ...string) (*bytes.Buffer, error) {
 	cmd.Stdout = stdout
 	cmd.Stderr = stderr
 	defer func(start time.Time) {
-		cfg.Logf("%s for %v, stderr: <<%s>>\n", time.Since(start), cmdDebugStr(cmd, args...), stderr)
+		cfg.Logf("%s for %v, stderr: <<%s>> stdout: <<%s>>\n", time.Since(start), cmdDebugStr(cmd, args...), stderr, stdout)
 	}(time.Now())
 
 	if err := cmd.Run(); err != nil {
@@ -896,7 +1013,7 @@ func invokeGo(cfg *Config, args ...string) (*bytes.Buffer, error) {
 			//    (the Graphic characters without spaces) and may also exclude the
 			//    characters !"#$%&'()*,:;<=>?[\]^`{|} and the Unicode replacement character U+FFFD.
 			return unicode.IsOneOf([]*unicode.RangeTable{unicode.L, unicode.M, unicode.N, unicode.P, unicode.S}, r) &&
-				strings.IndexRune("!\"#$%&'()*,:;<=>?[\\]^`{|}\uFFFD", r) == -1
+				!strings.ContainsRune("!\"#$%&'()*,:;<=>?[\\]^`{|}\uFFFD", r)
 		}
 		if len(stderr.String()) > 0 && strings.HasPrefix(stderr.String(), "# ") {
 			if strings.HasPrefix(strings.TrimLeftFunc(stderr.String()[len("# "):], isPkgPathRune), "\n") {
@@ -947,13 +1064,24 @@ func invokeGo(cfg *Config, args ...string) (*bytes.Buffer, error) {
 				// TODO(matloob): command-line-arguments isn't correct here.
 				"command-line-arguments", strings.Trim(stderr.String(), "\n"))
 			return bytes.NewBufferString(output), nil
+		}
 
+		// Another variation of the previous error
+		if len(stderr.String()) > 0 && strings.Contains(stderr.String(), "outside module root") {
+			output := fmt.Sprintf(`{"ImportPath": %q,"Incomplete": true,"Error": {"Pos": "","Err": %q}}`,
+				// TODO(matloob): command-line-arguments isn't correct here.
+				"command-line-arguments", strings.Trim(stderr.String(), "\n"))
+			return bytes.NewBufferString(output), nil
 		}
 
 		// Workaround for an instance of golang.org/issue/26755: go list -e  will return a non-zero exit
 		// status if there's a dependency on a package that doesn't exist. But it should return
 		// a zero exit status and set an error on that package.
 		if len(stderr.String()) > 0 && strings.Contains(stderr.String(), "no Go files in") {
+			// Don't clobber stdout if `go list` actually returned something.
+			if len(stdout.String()) > 0 {
+				return stdout, nil
+			}
 			// try to extract package name from string
 			stderrStr := stderr.String()
 			var importPath string
@@ -987,12 +1115,6 @@ func invokeGo(cfg *Config, args ...string) (*bytes.Buffer, error) {
 	if len(stderr.Bytes()) != 0 && os.Getenv("GOPACKAGESPRINTGOLISTERRORS") != "" {
 		fmt.Fprintf(os.Stderr, "%s stderr: <<%s>>\n", cmdDebugStr(cmd, args...), stderr)
 	}
-
-	// debugging
-	if false {
-		fmt.Fprintf(os.Stderr, "%s stdout: <<%s>>\n", cmdDebugStr(cmd, args...), stdout)
-	}
-
 	return stdout, nil
 }
 

vendor/golang.org/x/tools/go/packages/golist_overlay.go

@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"go/parser"
 	"go/token"
-	"path"
 	"path/filepath"
 	"strconv"
 	"strings"
@@ -87,26 +86,10 @@ func processGolistOverlay(cfg *Config, response *responseDeduper, rootDirs func(
 		if pkg == nil {
 			// Try to find the module or gopath dir the file is contained in.
 			// Then for modules, add the module opath to the beginning.
-			var pkgPath string
-			for rdir, rpath := range rootDirs().rootDirs {
-				// TODO(matloob): This doesn't properly handle symlinks.
-				r, err := filepath.Rel(rdir, dir)
-				if err != nil {
-					continue
-				}
-				pkgPath = filepath.ToSlash(r)
-				if rpath != "" {
-					pkgPath = path.Join(rpath, pkgPath)
-				}
-				// We only create one new package even it can belong in multiple modules or GOPATH entries.
-				// This is okay because tools (such as the LSP) that use overlays will recompute the overlay
-				// once the file is saved, and golist will do the right thing.
-				// TODO(matloob): Implement module tiebreaking?
+			pkgPath, ok := getPkgPath(cfg, dir, rootDirs)
+			if !ok {
 				break
 			}
-			if pkgPath == "" {
-				continue
-			}
 			isXTest := strings.HasSuffix(pkgName, "_test")
 			if isXTest {
 				pkgPath += "_test"

vendor/golang.org/x/tools/go/packages/loadmode_string.go

@@ -0,0 +1,57 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package packages
+
+import (
+	"fmt"
+	"strings"
+)
+
+var allModes = []LoadMode{
+	NeedName,
+	NeedFiles,
+	NeedCompiledGoFiles,
+	NeedImports,
+	NeedDeps,
+	NeedExportsFile,
+	NeedTypes,
+	NeedSyntax,
+	NeedTypesInfo,
+	NeedTypesSizes,
+}
+
+var modeStrings = []string{
+	"NeedName",
+	"NeedFiles",
+	"NeedCompiledGoFiles",
+	"NeedImports",
+	"NeedDeps",
+	"NeedExportsFile",
+	"NeedTypes",
+	"NeedSyntax",
+	"NeedTypesInfo",
+	"NeedTypesSizes",
+}
+
+func (mod LoadMode) String() string {
+	m := mod
+	if m == 0 {
+		return fmt.Sprintf("LoadMode(0)")
+	}
+	var out []string
+	for i, x := range allModes {
+		if x > m {
+			break
+		}
+		if (m & x) != 0 {
+			out = append(out, modeStrings[i])
+			m = m ^ x
+		}
+	}
+	if m != 0 {
+		out = append(out, "Unknown")
+	}
+	return fmt.Sprintf("LoadMode(%s)", strings.Join(out, "|"))
+}

vendor/golang.org/x/tools/go/packages/packages.go

@@ -467,7 +467,7 @@ func newLoader(cfg *Config) *loader {
 	ld.requestedMode = ld.Mode
 	ld.Mode = impliedLoadMode(ld.Mode)
 
-	if ld.Mode&NeedTypes != 0 {
+	if ld.Mode&NeedTypes != 0 || ld.Mode&NeedSyntax != 0 {
 		if ld.Fset == nil {
 			ld.Fset = token.NewFileSet()
 		}
@@ -609,9 +609,9 @@ func (ld *loader) refine(roots []string, list ...*Package) ([]*Package, error) {
 			}
 		}
 	}
-	// Load type data if needed, starting at
+	// Load type data and syntax if needed, starting at
 	// the initial packages (roots of the import DAG).
-	if ld.Mode&NeedTypes != 0 {
+	if ld.Mode&NeedTypes != 0 || ld.Mode&NeedSyntax != 0 {
 		var wg sync.WaitGroup
 		for _, lpkg := range initial {
 			wg.Add(1)
@@ -770,7 +770,7 @@ func (ld *loader) loadPackage(lpkg *loaderPackage) {
 		lpkg.Errors = append(lpkg.Errors, errs...)
 	}
 
-	if len(lpkg.CompiledGoFiles) == 0 && lpkg.ExportFile != "" {
+	if ld.Config.Mode&NeedTypes != 0 && len(lpkg.CompiledGoFiles) == 0 && lpkg.ExportFile != "" {
 		// The config requested loading sources and types, but sources are missing.
 		// Add an error to the package and fall back to loading from export data.
 		appendError(Error{"-", fmt.Sprintf("sources missing for package %s", lpkg.ID), ParseError})
@@ -784,6 +784,9 @@ func (ld *loader) loadPackage(lpkg *loaderPackage) {
 	}
 
 	lpkg.Syntax = files
+	if ld.Config.Mode&NeedTypes == 0 {
+		return
+	}
 
 	lpkg.TypesInfo = &types.Info{
 		Types:      make(map[ast.Expr]types.TypeAndValue),