@ -14,10 +14,10 @@ import (
"strconv"
"strings"
"sync"
"time"
"golang.org/x/tools/internal/gopathwalk"
"golang.org/x/tools/internal/module"
"golang.org/x/tools/internal/semver"
)
// ModuleResolver implements resolver for modules using the go command as little
@ -25,38 +25,81 @@ import (
type ModuleResolver struct {
env * ProcessEnv
moduleCacheDir string
dummyVendorMod * ModuleJSON // If vendoring is enabled, the pseudo-module that represents the /vendor directory.
Initialized bool
Main * ModuleJSON
ModsByModPath [ ] * ModuleJSON // All modules, ordered by # of path components in module Path...
ModsByDir [ ] * ModuleJSON // ...or Dir.
// moduleCacheInfo stores information about the module cache.
moduleCacheInfo * moduleCacheInfo
// moduleCacheCache stores information about the module cache.
moduleCacheCache * dirInfoCache
otherCache * dirInfoCache
}
type ModuleJSON struct {
Path string // module path
Version string // module version
Versions [ ] string // available module versions (with -versions)
Replace * ModuleJSON // replaced by this module
Time * time . Time // time version was created
Update * ModuleJSON // available update, if any (with -u)
Main bool // is this the main module?
Indirect bool // is this module only an indirect dependency of main module?
Dir string // directory holding files for this module, if any
GoMod string // path to go.mod file for this module, if any
Error * ModuleErrorJSON // error loading module
}
type ModuleErrorJSON struct {
Err string // the error itself
Path string // module path
Replace * ModuleJSON // replaced by this module
Main bool // is this the main module?
Dir string // directory holding files for this module, if any
GoMod string // path to go.mod file for this module, if any
GoVersion string // go version used in module
}
func ( r * ModuleResolver ) init ( ) error {
if r . Initialized {
return nil
}
mainMod , vendorEnabled , err := vendorEnabled ( r . env )
if err != nil {
return err
}
if mainMod != nil && vendorEnabled {
// Vendor mode is on, so all the non-Main modules are irrelevant,
// and we need to search /vendor for everything.
r . Main = mainMod
r . dummyVendorMod = & ModuleJSON {
Path : "" ,
Dir : filepath . Join ( mainMod . Dir , "vendor" ) ,
}
r . ModsByModPath = [ ] * ModuleJSON { mainMod , r . dummyVendorMod }
r . ModsByDir = [ ] * ModuleJSON { mainMod , r . dummyVendorMod }
} else {
// Vendor mode is off, so run go list -m ... to find everything.
r . initAllMods ( )
}
r . moduleCacheDir = filepath . Join ( filepath . SplitList ( r . env . GOPATH ) [ 0 ] , "/pkg/mod" )
sort . Slice ( r . ModsByModPath , func ( i , j int ) bool {
count := func ( x int ) int {
return strings . Count ( r . ModsByModPath [ x ] . Path , "/" )
}
return count ( j ) < count ( i ) // descending order
} )
sort . Slice ( r . ModsByDir , func ( i , j int ) bool {
count := func ( x int ) int {
return strings . Count ( r . ModsByDir [ x ] . Dir , "/" )
}
return count ( j ) < count ( i ) // descending order
} )
if r . moduleCacheCache == nil {
r . moduleCacheCache = & dirInfoCache {
dirs : map [ string ] * directoryPackageInfo { } ,
}
}
if r . otherCache == nil {
r . otherCache = & dirInfoCache {
dirs : map [ string ] * directoryPackageInfo { } ,
}
}
r . Initialized = true
return nil
}
func ( r * ModuleResolver ) initAllMods ( ) error {
stdout , err := r . env . invokeGo ( "list" , "-m" , "-json" , "..." )
if err != nil {
return err
@ -79,28 +122,21 @@ func (r *ModuleResolver) init() error {
r . Main = mod
}
}
return nil
}
sort . Slice ( r . ModsByModPath , func ( i , j int ) bool {
count := func ( x int ) int {
return strings . Count ( r . ModsByModPath [ x ] . Path , "/" )
}
return count ( j ) < count ( i ) // descending order
} )
sort . Slice ( r . ModsByDir , func ( i , j int ) bool {
count := func ( x int ) int {
return strings . Count ( r . ModsByDir [ x ] . Dir , "/" )
}
return count ( j ) < count ( i ) // descending order
} )
if r . moduleCacheInfo == nil {
r . moduleCacheInfo = & moduleCacheInfo {
modCacheDirInfo : make ( map [ string ] * directoryPackageInfo ) ,
}
func ( r * ModuleResolver ) ClearForNewScan ( ) {
r . otherCache = & dirInfoCache {
dirs : map [ string ] * directoryPackageInfo { } ,
}
}
r . Initialized = true
return nil
func ( r * ModuleResolver ) ClearForNewMod ( ) {
env := r . env
* r = ModuleResolver {
env : env ,
}
r . init ( )
}
// findPackage returns the module and directory that contains the package at
@ -118,22 +154,31 @@ func (r *ModuleResolver) findPackage(importPath string) (*ModuleJSON, string) {
continue
}
if info , ok := r . moduleCacheInfo . Load ( pkgDir ) ; ok {
if packageScann ed, err := info . reachedStatus ( directoryScanned ) ; packageScann ed{
if info , ok := r . cache Load( pkgDir ) ; ok {
if load ed, err := info . reachedStatus ( nameLoaded ) ; load ed{
if err != nil {
// There was some error with scanning this directory.
// It does not contain a valid package.
continue
continue // No package in this dir.
}
return m , pkgDir
}
if scanned , err := info . reachedStatus ( directoryScanned ) ; scanned && err != nil {
continue // Dir is unreadable, etc.
}
// This is slightly wrong: a directory doesn't have to have an
// importable package to count as a package for package-to-module
// resolution. package main or _test files should count but
// don't.
// TODO(heschi): fix this.
if _ , err := r . cachePackageName ( info ) ; err == nil {
return m , pkgDir
}
}
// Not cached. Read the filesystem.
pkgFiles , err := ioutil . ReadDir ( pkgDir )
if err != nil {
continue
}
// A module only contains a package if it has buildable go
// files in that directory. If not, it could be provided by an
// outer module. See #29736.
@ -146,6 +191,40 @@ func (r *ModuleResolver) findPackage(importPath string) (*ModuleJSON, string) {
return nil , ""
}
func ( r * ModuleResolver ) cacheLoad ( dir string ) ( directoryPackageInfo , bool ) {
if info , ok := r . moduleCacheCache . Load ( dir ) ; ok {
return info , ok
}
return r . otherCache . Load ( dir )
}
func ( r * ModuleResolver ) cacheStore ( info directoryPackageInfo ) {
if info . rootType == gopathwalk . RootModuleCache {
r . moduleCacheCache . Store ( info . dir , info )
} else {
r . otherCache . Store ( info . dir , info )
}
}
func ( r * ModuleResolver ) cacheKeys ( ) [ ] string {
return append ( r . moduleCacheCache . Keys ( ) , r . otherCache . Keys ( ) ... )
}
// cachePackageName caches the package name for a dir already in the cache.
func ( r * ModuleResolver ) cachePackageName ( info directoryPackageInfo ) ( directoryPackageInfo , error ) {
if info . rootType == gopathwalk . RootModuleCache {
return r . moduleCacheCache . CachePackageName ( info )
}
return r . otherCache . CachePackageName ( info )
}
func ( r * ModuleResolver ) cacheExports ( ctx context . Context , env * ProcessEnv , info directoryPackageInfo ) ( string , [ ] string , error ) {
if info . rootType == gopathwalk . RootModuleCache {
return r . moduleCacheCache . CacheExports ( ctx , env , info )
}
return r . otherCache . CacheExports ( ctx , env , info )
}
// findModuleByDir returns the module that contains dir, or nil if no such
// module is in scope.
func ( r * ModuleResolver ) findModuleByDir ( dir string ) * ModuleJSON {
@ -184,28 +263,45 @@ func (r *ModuleResolver) dirIsNestedModule(dir string, mod *ModuleJSON) bool {
// so it cannot be a nested module.
return false
}
mf := r . findModFile ( dir )
if mf == "" {
if mod != nil && mod == r . dummyVendorMod {
// The /vendor pseudomodule is flattened and doesn't actually count.
return false
}
return filepath . Dir ( mf ) != mod . Dir
modDir , _ := r . modInfo ( dir )
if modDir == "" {
return false
}
return modDir != mod . Dir
}
func ( r * ModuleResolver ) findModFile ( dir string ) string {
func ( r * ModuleResolver ) modInfo ( dir string ) ( modDir string , modName string ) {
readModName := func ( modFile string ) string {
modBytes , err := ioutil . ReadFile ( modFile )
if err != nil {
return ""
}
return modulePath ( modBytes )
}
if r . dirInModuleCache ( dir ) {
matches := modCacheRegexp . FindStringSubmatch ( dir )
index := strings . Index ( dir , matches [ 1 ] + "@" + matches [ 2 ] )
return filepath . Join ( dir [ : index ] , matches [ 1 ] + "@" + matches [ 2 ] , "go.mod" )
modDir := filepath . Join ( dir [ : index ] , matches [ 1 ] + "@" + matches [ 2 ] )
return modDir , readModName ( filepath . Join ( modDir , "go.mod" ) )
}
for {
if info , ok := r . cacheLoad ( dir ) ; ok {
return info . moduleDir , info . moduleName
}
f := filepath . Join ( dir , "go.mod" )
info , err := os . Stat ( f )
if err == nil && ! info . IsDir ( ) {
return f
return dir , readModName ( f )
}
d := filepath . Dir ( dir )
if len ( d ) >= len ( dir ) {
return "" // reached top of file system, no go.mod
return "" , "" // reached top of file system, no go.mod
}
dir = d
}
@ -237,7 +333,7 @@ func (r *ModuleResolver) loadPackageNames(importPaths []string, srcDir string) (
return names , nil
}
func ( r * ModuleResolver ) scan ( _ references ) ( [ ] * pkg , error ) {
func ( r * ModuleResolver ) scan ( _ references , loadNames bool , exclude [ ] gopathwalk . RootType ) ( [ ] * pkg , error ) {
if err := r . init ( ) ; err != nil {
return nil , err
}
@ -249,35 +345,30 @@ func (r *ModuleResolver) scan(_ references) ([]*pkg, error) {
if r . Main != nil {
roots = append ( roots , gopathwalk . Root { r . Main . Dir , gopathwalk . RootCurrentModule } )
}
if r . moduleCacheDir == "" {
r . moduleCacheDir = filepath . Join ( filepath . SplitList ( r . env . GOPATH ) [ 0 ] , "/pkg/mod" )
}
roots = append ( roots , gopathwalk . Root { r . moduleCacheDir , gopathwalk . RootModuleCache } )
// Walk replace targets, just in case they're not in any of the above.
for _ , mod := range r . ModsByModPath {
if mod . Replace != nil {
roots = append ( roots , gopathwalk . Root { mod . Dir , gopathwalk . RootOther } )
if r . dummyVendorMod != nil {
roots = append ( roots , gopathwalk . Root { r . dummyVendorMod . Dir , gopathwalk . RootOther } )
} else {
roots = append ( roots , gopathwalk . Root { r . moduleCacheDir , gopathwalk . RootModuleCache } )
// Walk replace targets, just in case they're not in any of the above.
for _ , mod := range r . ModsByModPath {
if mod . Replace != nil {
roots = append ( roots , gopathwalk . Root { mod . Dir , gopathwalk . RootOther } )
}
}
}
roots = filterRoots ( roots , exclude )
var result [ ] * pkg
dupCheck := make ( map [ string ] bool )
var mu sync . Mutex
// Packages in the module cache are immutable. If we have
// already seen this package on a previous scan of the module
// cache, return that result.
// We assume cached directories have not changed. We can skip them and their
// children.
skip := func ( root gopathwalk . Root , dir string ) bool {
mu . Lock ( )
defer mu . Unlock ( )
// If we have already processed this directory on this walk, skip it.
if _ , dup := dupCheck [ dir ] ; dup {
return true
}
// If we have saved this directory information, skip it.
info , ok := r . moduleCacheInfo . Load ( dir )
info , ok := r . cacheLoad ( dir )
if ! ok {
return false
}
@ -288,44 +379,19 @@ func (r *ModuleResolver) scan(_ references) ([]*pkg, error) {
return packageScanned
}
// Add anything new to the cache. We'll process everything in it below.
add := func ( root gopathwalk . Root , dir string ) {
mu . Lock ( )
defer mu . Unlock ( )
if _ , dup := dupCheck [ dir ] ; dup {
return
}
info , err := r . scanDirForPackage ( root , dir )
if err != nil {
return
}
if root . Type == gopathwalk . RootModuleCache {
// Save this package information in the cache and return.
// Packages from the module cache are added after Walk.
r . moduleCacheInfo . Store ( dir , info )
return
}
// Skip this package if there was an error loading package info.
if info . err != nil {
return
}
// The rest of this function canonicalizes the packages using the results
// of initializing the resolver from 'go list -m'.
res , err := r . canonicalize ( info . nonCanonicalImportPath , info . dir , info . needsReplace )
if err != nil {
return
}
result = append ( result , res )
r . cacheStore ( r . scanDirForPackage ( root , dir ) )
}
gopathwalk . WalkSkip ( roots , add , skip , gopathwalk . Options { Debug : r . env . Debug , ModulesEnabled : true } )
// Add the packages from the modules in the mod cache that were skipped .
for _ , dir := range r . moduleCacheInfo . Keys ( ) {
info , ok := r . moduleCacheInfo . Load ( dir )
// Everything we already had, and everything new, is now in the cache.
for _ , dir := range r . cacheKeys ( ) {
info , ok := r . cacheLoad ( dir )
if ! ok {
continue
}
@ -335,7 +401,15 @@ func (r *ModuleResolver) scan(_ references) ([]*pkg, error) {
continue
}
res , err := r . canonicalize ( info . nonCanonicalImportPath , info . dir , info . needsReplace )
// If we want package names, make sure the cache has them.
if loadNames {
var err error
if info , err = r . cachePackageName ( info ) ; err != nil {
continue
}
}
res , err := r . canonicalize ( info )
if err != nil {
continue
}
@ -347,52 +421,70 @@ func (r *ModuleResolver) scan(_ references) ([]*pkg, error) {
// canonicalize gets the result of canonicalizing the packages using the results
// of initializing the resolver from 'go list -m'.
func ( r * ModuleResolver ) canonicalize ( importPath , dir string , needsReplace bool ) ( res * pkg , err error ) {
func ( r * ModuleResolver ) canonicalize ( info directoryPackageInfo ) ( * pkg , error ) {
// Packages in GOROOT are already canonical, regardless of the std/cmd modules.
if info . rootType == gopathwalk . RootGOROOT {
return & pkg {
importPathShort : info . nonCanonicalImportPath ,
dir : info . dir ,
packageName : path . Base ( info . nonCanonicalImportPath ) ,
relevance : 0 ,
} , nil
}
importPath := info . nonCanonicalImportPath
relevance := 2
// Check if the directory is underneath a module that's in scope.
if mod := r . findModuleByDir ( dir ) ; mod != nil {
if mod := r . findModuleByDir ( info . dir ) ; mod != nil {
relevance = 1
// It is. If dir is the target of a replace directive,
// our guessed import path is wrong. Use the real one.
if mod . Dir == dir {
if mod . Dir == info . dir {
importPath = mod . Path
} else {
dirInMod := dir [ len ( mod . Dir ) + len ( "/" ) : ]
dirInMod := info . dir [ len ( mod . Dir ) + len ( "/" ) : ]
importPath = path . Join ( mod . Path , filepath . ToSlash ( dirInMod ) )
}
} else if needsReplace {
return nil , fmt . Errorf ( "needed this package to be in scope: %s" , dir )
} else if info . needsReplace {
return nil , fmt . Errorf ( "package in %q is not valid without a replace statement" , info . dir )
}
res := & pkg {
importPathShort : importPath ,
dir : info . dir ,
packageName : info . packageName , // may not be populated if the caller didn't ask for it
relevance : relevance ,
}
// We may have discovered a package that has a different version
// in scope already. Canonicalize to that one if possible.
if _ , canonicalDir := r . findPackage ( importPath ) ; canonicalDir != "" {
dir = canonicalDir
res . dir = canonicalDir
}
return & pkg {
importPathShort : VendorlessPath ( importPath ) ,
dir : dir ,
} , nil
return res , nil
}
func ( r * ModuleResolver ) loadExports ( ctx context . Context , expectPackage string , pkg * pkg ) ( map [ string ] bool , error ) {
func ( r * ModuleResolver ) loadExports ( ctx context . Context , pkg * pkg ) ( string , [ ] string , error ) {
if err := r . init ( ) ; err != nil {
return nil , err
return "" , nil , err
}
return loadExportsFromFiles ( ctx , r . env , expectPackage , pkg . dir )
if info , ok := r . cacheLoad ( pkg . dir ) ; ok {
return r . cacheExports ( ctx , r . env , info )
}
return loadExportsFromFiles ( ctx , r . env , pkg . dir )
}
func ( r * ModuleResolver ) scanDirForPackage ( root gopathwalk . Root , dir string ) ( directoryPackageInfo , error ) {
func ( r * ModuleResolver ) scanDirForPackage ( root gopathwalk . Root , dir string ) directoryPackageInfo {
subdir := ""
if dir != root . Path {
subdir = dir [ len ( root . Path ) + len ( "/" ) : ]
}
importPath := filepath . ToSlash ( subdir )
if strings . HasPrefix ( importPath , "vendor/" ) {
// Ignore vendor dirs. If -mod=vendor is on, then things
// should mostly just work, but when it's not vendor/
// is a mess. There's no easy way to tell if it's on.
// We can still find things in the mod cache and
// map them into /vendor when -mod=vendor is on.
return directoryPackageInfo { } , fmt . Errorf ( "vendor directory" )
// Only enter vendor directories if they're explicitly requested as a root.
return directoryPackageInfo {
status : directoryScanned ,
err : fmt . Errorf ( "unwanted vendor directory" ) ,
}
}
switch root . Type {
case gopathwalk . RootCurrentModule :
@ -403,7 +495,7 @@ func (r *ModuleResolver) scanDirForPackage(root gopathwalk.Root, dir string) (di
return directoryPackageInfo {
status : directoryScanned ,
err : fmt . Errorf ( "invalid module cache path: %v" , subdir ) ,
} , nil
}
}
modPath , err := module . DecodePath ( filepath . ToSlash ( matches [ 1 ] ) )
if err != nil {
@ -413,35 +505,34 @@ func (r *ModuleResolver) scanDirForPackage(root gopathwalk.Root, dir string) (di
return directoryPackageInfo {
status : directoryScanned ,
err : fmt . Errorf ( "decoding module cache path %q: %v" , subdir , err ) ,
} , nil
}
}
importPath = path . Join ( modPath , filepath . ToSlash ( matches [ 3 ] ) )
case gopathwalk . RootGOROOT :
importPath = subdir
}
modDir , modName := r . modInfo ( dir )
result := directoryPackageInfo {
status : directoryScanned ,
dir : dir ,
rootType : root . Type ,
nonCanonicalImportPath : importPath ,
needsReplace : false ,
moduleDir : modDir ,
moduleName : modName ,
}
if root . Type == gopathwalk . RootGOROOT {
// stdlib packages are always in scope, despite the confusing go.mod
return result , nil
return result
}
// Check that this package is not obviously impossible to import.
modFile := r . findModFile ( dir )
modBytes , err := ioutil . ReadFile ( modFile )
if err == nil && ! strings . HasPrefix ( importPath , modulePath ( modBytes ) ) {
if ! strings . HasPrefix ( importPath , modName ) {
// The module's declared path does not match
// its expected path. It probably needs a
// replace directive we don't have.
result . needsReplace = true
}
return result , nil
return result
}
// modCacheRegexp splits a path in a module cache into module, module version, and package.
@ -490,3 +581,63 @@ func modulePath(mod []byte) string {
}
return "" // missing module path
}
var modFlagRegexp = regexp . MustCompile ( ` -mod[ =](\w+) ` )
// vendorEnabled indicates if vendoring is enabled.
// Inspired by setDefaultBuildMod in modload/init.go
func vendorEnabled ( env * ProcessEnv ) ( * ModuleJSON , bool , error ) {
mainMod , go114 , err := getMainModuleAnd114 ( env )
if err != nil {
return nil , false , err
}
matches := modFlagRegexp . FindStringSubmatch ( env . GOFLAGS )
var modFlag string
if len ( matches ) != 0 {
modFlag = matches [ 1 ]
}
if modFlag != "" {
// Don't override an explicit '-mod=' argument.
return mainMod , modFlag == "vendor" , nil
}
if mainMod == nil || ! go114 {
return mainMod , false , nil
}
// Check 1.14's automatic vendor mode.
if fi , err := os . Stat ( filepath . Join ( mainMod . Dir , "vendor" ) ) ; err == nil && fi . IsDir ( ) {
if mainMod . GoVersion != "" && semver . Compare ( "v" + mainMod . GoVersion , "v1.14" ) >= 0 {
// The Go version is at least 1.14, and a vendor directory exists.
// Set -mod=vendor by default.
return mainMod , true , nil
}
}
return mainMod , false , nil
}
// getMainModuleAnd114 gets the main module's information and whether the
// go command in use is 1.14+. This is the information needed to figure out
// if vendoring should be enabled.
func getMainModuleAnd114 ( env * ProcessEnv ) ( * ModuleJSON , bool , error ) {
const format = ` { { . Path } }
{ { . Dir } }
{ { . GoMod } }
{ { . GoVersion } }
{ { range context . ReleaseTags } } { { if eq . "go1.14" } } { { . } } { { end } } { { end } }
`
stdout , err := env . invokeGo ( "list" , "-m" , "-f" , format )
if err != nil {
return nil , false , nil
}
lines := strings . Split ( stdout . String ( ) , "\n" )
if len ( lines ) < 5 {
return nil , false , fmt . Errorf ( "unexpected stdout: %q" , stdout )
}
mod := & ModuleJSON {
Path : lines [ 0 ] ,
Dir : lines [ 1 ] ,
GoMod : lines [ 2 ] ,
GoVersion : lines [ 3 ] ,
Main : true ,
}
return mod , lines [ 4 ] == "go1.14" , nil
}
zeripath
5 years ago
committed by
GitHub