| 1 | // Copyright 2013 The Go Authors. All rights reserved. |
|---|---|
| 2 | // Use of this source code is governed by a BSD-style |
| 3 | // license that can be found in the LICENSE file. |
| 4 | |
| 5 | package loader |
| 6 | |
| 7 | // See doc.go for package documentation and implementation notes. |
| 8 | |
| 9 | import ( |
| 10 | "errors" |
| 11 | "fmt" |
| 12 | "go/ast" |
| 13 | "go/build" |
| 14 | "go/parser" |
| 15 | "go/token" |
| 16 | "go/types" |
| 17 | "os" |
| 18 | "path/filepath" |
| 19 | "sort" |
| 20 | "strings" |
| 21 | "sync" |
| 22 | "time" |
| 23 | |
| 24 | "golang.org/x/tools/go/ast/astutil" |
| 25 | "golang.org/x/tools/go/internal/cgo" |
| 26 | "golang.org/x/tools/internal/typeparams" |
| 27 | ) |
| 28 | |
| 29 | var ignoreVendor build.ImportMode |
| 30 | |
| 31 | const trace = false // show timing info for type-checking |
| 32 | |
| 33 | // Config specifies the configuration for loading a whole program from |
| 34 | // Go source code. |
| 35 | // The zero value for Config is a ready-to-use default configuration. |
| 36 | type Config struct { |
| 37 | // Fset is the file set for the parser to use when loading the |
| 38 | // program. If nil, it may be lazily initialized by any |
| 39 | // method of Config. |
| 40 | Fset *token.FileSet |
| 41 | |
| 42 | // ParserMode specifies the mode to be used by the parser when |
| 43 | // loading source packages. |
| 44 | ParserMode parser.Mode |
| 45 | |
| 46 | // TypeChecker contains options relating to the type checker. |
| 47 | // |
| 48 | // The supplied IgnoreFuncBodies is not used; the effective |
| 49 | // value comes from the TypeCheckFuncBodies func below. |
| 50 | // The supplied Import function is not used either. |
| 51 | TypeChecker types.Config |
| 52 | |
| 53 | // TypeCheckFuncBodies is a predicate over package paths. |
| 54 | // A package for which the predicate is false will |
| 55 | // have its package-level declarations type checked, but not |
| 56 | // its function bodies; this can be used to quickly load |
| 57 | // dependencies from source. If nil, all func bodies are type |
| 58 | // checked. |
| 59 | TypeCheckFuncBodies func(path string) bool |
| 60 | |
| 61 | // If Build is non-nil, it is used to locate source packages. |
| 62 | // Otherwise &build.Default is used. |
| 63 | // |
| 64 | // By default, cgo is invoked to preprocess Go files that |
| 65 | // import the fake package "C". This behaviour can be |
| 66 | // disabled by setting CGO_ENABLED=0 in the environment prior |
| 67 | // to startup, or by setting Build.CgoEnabled=false. |
| 68 | Build *build.Context |
| 69 | |
| 70 | // The current directory, used for resolving relative package |
| 71 | // references such as "./go/loader". If empty, os.Getwd will be |
| 72 | // used instead. |
| 73 | Cwd string |
| 74 | |
| 75 | // If DisplayPath is non-nil, it is used to transform each |
| 76 | // file name obtained from Build.Import(). This can be used |
| 77 | // to prevent a virtualized build.Config's file names from |
| 78 | // leaking into the user interface. |
| 79 | DisplayPath func(path string) string |
| 80 | |
| 81 | // If AllowErrors is true, Load will return a Program even |
| 82 | // if some of the its packages contained I/O, parser or type |
| 83 | // errors; such errors are accessible via PackageInfo.Errors. If |
| 84 | // false, Load will fail if any package had an error. |
| 85 | AllowErrors bool |
| 86 | |
| 87 | // CreatePkgs specifies a list of non-importable initial |
| 88 | // packages to create. The resulting packages will appear in |
| 89 | // the corresponding elements of the Program.Created slice. |
| 90 | CreatePkgs []PkgSpec |
| 91 | |
| 92 | // ImportPkgs specifies a set of initial packages to load. |
| 93 | // The map keys are package paths. |
| 94 | // |
| 95 | // The map value indicates whether to load tests. If true, Load |
| 96 | // will add and type-check two lists of files to the package: |
| 97 | // non-test files followed by in-package *_test.go files. In |
| 98 | // addition, it will append the external test package (if any) |
| 99 | // to Program.Created. |
| 100 | ImportPkgs map[string]bool |
| 101 | |
| 102 | // FindPackage is called during Load to create the build.Package |
| 103 | // for a given import path from a given directory. |
| 104 | // If FindPackage is nil, (*build.Context).Import is used. |
| 105 | // A client may use this hook to adapt to a proprietary build |
| 106 | // system that does not follow the "go build" layout |
| 107 | // conventions, for example. |
| 108 | // |
| 109 | // It must be safe to call concurrently from multiple goroutines. |
| 110 | FindPackage func(ctxt *build.Context, importPath, fromDir string, mode build.ImportMode) (*build.Package, error) |
| 111 | |
| 112 | // AfterTypeCheck is called immediately after a list of files |
| 113 | // has been type-checked and appended to info.Files. |
| 114 | // |
| 115 | // This optional hook function is the earliest opportunity for |
| 116 | // the client to observe the output of the type checker, |
| 117 | // which may be useful to reduce analysis latency when loading |
| 118 | // a large program. |
| 119 | // |
| 120 | // The function is permitted to modify info.Info, for instance |
| 121 | // to clear data structures that are no longer needed, which can |
| 122 | // dramatically reduce peak memory consumption. |
| 123 | // |
| 124 | // The function may be called twice for the same PackageInfo: |
| 125 | // once for the files of the package and again for the |
| 126 | // in-package test files. |
| 127 | // |
| 128 | // It must be safe to call concurrently from multiple goroutines. |
| 129 | AfterTypeCheck func(info *PackageInfo, files []*ast.File) |
| 130 | } |
| 131 | |
| 132 | // A PkgSpec specifies a non-importable package to be created by Load. |
| 133 | // Files are processed first, but typically only one of Files and |
| 134 | // Filenames is provided. The path needn't be globally unique. |
| 135 | // |
| 136 | // For vendoring purposes, the package's directory is the one that |
| 137 | // contains the first file. |
| 138 | type PkgSpec struct { |
| 139 | Path string // package path ("" => use package declaration) |
| 140 | Files []*ast.File // ASTs of already-parsed files |
| 141 | Filenames []string // names of files to be parsed |
| 142 | } |
| 143 | |
| 144 | // A Program is a Go program loaded from source as specified by a Config. |
| 145 | type Program struct { |
| 146 | Fset *token.FileSet // the file set for this program |
| 147 | |
| 148 | // Created[i] contains the initial package whose ASTs or |
| 149 | // filenames were supplied by Config.CreatePkgs[i], followed by |
| 150 | // the external test package, if any, of each package in |
| 151 | // Config.ImportPkgs ordered by ImportPath. |
| 152 | // |
| 153 | // NOTE: these files must not import "C". Cgo preprocessing is |
| 154 | // only performed on imported packages, not ad hoc packages. |
| 155 | // |
| 156 | // TODO(adonovan): we need to copy and adapt the logic of |
| 157 | // goFilesPackage (from $GOROOT/src/cmd/go/build.go) and make |
| 158 | // Config.Import and Config.Create methods return the same kind |
| 159 | // of entity, essentially a build.Package. |
| 160 | // Perhaps we can even reuse that type directly. |
| 161 | Created []*PackageInfo |
| 162 | |
| 163 | // Imported contains the initially imported packages, |
| 164 | // as specified by Config.ImportPkgs. |
| 165 | Imported map[string]*PackageInfo |
| 166 | |
| 167 | // AllPackages contains the PackageInfo of every package |
| 168 | // encountered by Load: all initial packages and all |
| 169 | // dependencies, including incomplete ones. |
| 170 | AllPackages map[*types.Package]*PackageInfo |
| 171 | |
| 172 | // importMap is the canonical mapping of package paths to |
| 173 | // packages. It contains all Imported initial packages, but not |
| 174 | // Created ones, and all imported dependencies. |
| 175 | importMap map[string]*types.Package |
| 176 | } |
| 177 | |
| 178 | // PackageInfo holds the ASTs and facts derived by the type-checker |
| 179 | // for a single package. |
| 180 | // |
| 181 | // Not mutated once exposed via the API. |
| 182 | type PackageInfo struct { |
| 183 | Pkg *types.Package |
| 184 | Importable bool // true if 'import "Pkg.Path()"' would resolve to this |
| 185 | TransitivelyErrorFree bool // true if Pkg and all its dependencies are free of errors |
| 186 | Files []*ast.File // syntax trees for the package's files |
| 187 | Errors []error // non-nil if the package had errors |
| 188 | types.Info // type-checker deductions. |
| 189 | dir string // package directory |
| 190 | |
| 191 | checker *types.Checker // transient type-checker state |
| 192 | errorFunc func(error) |
| 193 | } |
| 194 | |
| 195 | func (info *PackageInfo) String() string { return info.Pkg.Path() } |
| 196 | |
| 197 | func (info *PackageInfo) appendError(err error) { |
| 198 | if info.errorFunc != nil { |
| 199 | info.errorFunc(err) |
| 200 | } else { |
| 201 | fmt.Fprintln(os.Stderr, err) |
| 202 | } |
| 203 | info.Errors = append(info.Errors, err) |
| 204 | } |
| 205 | |
| 206 | func (conf *Config) fset() *token.FileSet { |
| 207 | if conf.Fset == nil { |
| 208 | conf.Fset = token.NewFileSet() |
| 209 | } |
| 210 | return conf.Fset |
| 211 | } |
| 212 | |
| 213 | // ParseFile is a convenience function (intended for testing) that invokes |
| 214 | // the parser using the Config's FileSet, which is initialized if nil. |
| 215 | // |
| 216 | // src specifies the parser input as a string, []byte, or io.Reader, and |
| 217 | // filename is its apparent name. If src is nil, the contents of |
| 218 | // filename are read from the file system. |
| 219 | func (conf *Config) ParseFile(filename string, src interface{}) (*ast.File, error) { |
| 220 | // TODO(adonovan): use conf.build() etc like parseFiles does. |
| 221 | return parser.ParseFile(conf.fset(), filename, src, conf.ParserMode) |
| 222 | } |
| 223 | |
| 224 | // FromArgsUsage is a partial usage message that applications calling |
| 225 | // FromArgs may wish to include in their -help output. |
| 226 | const FromArgsUsage = ` |
| 227 | <args> is a list of arguments denoting a set of initial packages. |
| 228 | It may take one of two forms: |
| 229 | |
| 230 | 1. A list of *.go source files. |
| 231 | |
| 232 | All of the specified files are loaded, parsed and type-checked |
| 233 | as a single package. All the files must belong to the same directory. |
| 234 | |
| 235 | 2. A list of import paths, each denoting a package. |
| 236 | |
| 237 | The package's directory is found relative to the $GOROOT and |
| 238 | $GOPATH using similar logic to 'go build', and the *.go files in |
| 239 | that directory are loaded, parsed and type-checked as a single |
| 240 | package. |
| 241 | |
| 242 | In addition, all *_test.go files in the directory are then loaded |
| 243 | and parsed. Those files whose package declaration equals that of |
| 244 | the non-*_test.go files are included in the primary package. Test |
| 245 | files whose package declaration ends with "_test" are type-checked |
| 246 | as another package, the 'external' test package, so that a single |
| 247 | import path may denote two packages. (Whether this behaviour is |
| 248 | enabled is tool-specific, and may depend on additional flags.) |
| 249 | |
| 250 | A '--' argument terminates the list of packages. |
| 251 | ` |
| 252 | |
| 253 | // FromArgs interprets args as a set of initial packages to load from |
| 254 | // source and updates the configuration. It returns the list of |
| 255 | // unconsumed arguments. |
| 256 | // |
| 257 | // It is intended for use in command-line interfaces that require a |
| 258 | // set of initial packages to be specified; see FromArgsUsage message |
| 259 | // for details. |
| 260 | // |
| 261 | // Only superficial errors are reported at this stage; errors dependent |
| 262 | // on I/O are detected during Load. |
| 263 | func (conf *Config) FromArgs(args []string, xtest bool) ([]string, error) { |
| 264 | var rest []string |
| 265 | for i, arg := range args { |
| 266 | if arg == "--" { |
| 267 | rest = args[i+1:] |
| 268 | args = args[:i] |
| 269 | break // consume "--" and return the remaining args |
| 270 | } |
| 271 | } |
| 272 | |
| 273 | if len(args) > 0 && strings.HasSuffix(args[0], ".go") { |
| 274 | // Assume args is a list of a *.go files |
| 275 | // denoting a single ad hoc package. |
| 276 | for _, arg := range args { |
| 277 | if !strings.HasSuffix(arg, ".go") { |
| 278 | return nil, fmt.Errorf("named files must be .go files: %s", arg) |
| 279 | } |
| 280 | } |
| 281 | conf.CreateFromFilenames("", args...) |
| 282 | } else { |
| 283 | // Assume args are directories each denoting a |
| 284 | // package and (perhaps) an external test, iff xtest. |
| 285 | for _, arg := range args { |
| 286 | if xtest { |
| 287 | conf.ImportWithTests(arg) |
| 288 | } else { |
| 289 | conf.Import(arg) |
| 290 | } |
| 291 | } |
| 292 | } |
| 293 | |
| 294 | return rest, nil |
| 295 | } |
| 296 | |
| 297 | // CreateFromFilenames is a convenience function that adds |
| 298 | // a conf.CreatePkgs entry to create a package of the specified *.go |
| 299 | // files. |
| 300 | func (conf *Config) CreateFromFilenames(path string, filenames ...string) { |
| 301 | conf.CreatePkgs = append(conf.CreatePkgs, PkgSpec{Path: path, Filenames: filenames}) |
| 302 | } |
| 303 | |
| 304 | // CreateFromFiles is a convenience function that adds a conf.CreatePkgs |
| 305 | // entry to create package of the specified path and parsed files. |
| 306 | func (conf *Config) CreateFromFiles(path string, files ...*ast.File) { |
| 307 | conf.CreatePkgs = append(conf.CreatePkgs, PkgSpec{Path: path, Files: files}) |
| 308 | } |
| 309 | |
| 310 | // ImportWithTests is a convenience function that adds path to |
| 311 | // ImportPkgs, the set of initial source packages located relative to |
| 312 | // $GOPATH. The package will be augmented by any *_test.go files in |
| 313 | // its directory that contain a "package x" (not "package x_test") |
| 314 | // declaration. |
| 315 | // |
| 316 | // In addition, if any *_test.go files contain a "package x_test" |
| 317 | // declaration, an additional package comprising just those files will |
| 318 | // be added to CreatePkgs. |
| 319 | func (conf *Config) ImportWithTests(path string) { conf.addImport(path, true) } |
| 320 | |
| 321 | // Import is a convenience function that adds path to ImportPkgs, the |
| 322 | // set of initial packages that will be imported from source. |
| 323 | func (conf *Config) Import(path string) { conf.addImport(path, false) } |
| 324 | |
| 325 | func (conf *Config) addImport(path string, tests bool) { |
| 326 | if path == "C" { |
| 327 | return // ignore; not a real package |
| 328 | } |
| 329 | if conf.ImportPkgs == nil { |
| 330 | conf.ImportPkgs = make(map[string]bool) |
| 331 | } |
| 332 | conf.ImportPkgs[path] = conf.ImportPkgs[path] || tests |
| 333 | } |
| 334 | |
| 335 | // PathEnclosingInterval returns the PackageInfo and ast.Node that |
| 336 | // contain source interval [start, end), and all the node's ancestors |
| 337 | // up to the AST root. It searches all ast.Files of all packages in prog. |
| 338 | // exact is defined as for astutil.PathEnclosingInterval. |
| 339 | // |
| 340 | // The zero value is returned if not found. |
| 341 | func (prog *Program) PathEnclosingInterval(start, end token.Pos) (pkg *PackageInfo, path []ast.Node, exact bool) { |
| 342 | for _, info := range prog.AllPackages { |
| 343 | for _, f := range info.Files { |
| 344 | if f.Pos() == token.NoPos { |
| 345 | // This can happen if the parser saw |
| 346 | // too many errors and bailed out. |
| 347 | // (Use parser.AllErrors to prevent that.) |
| 348 | continue |
| 349 | } |
| 350 | if !tokenFileContainsPos(prog.Fset.File(f.Pos()), start) { |
| 351 | continue |
| 352 | } |
| 353 | if path, exact := astutil.PathEnclosingInterval(f, start, end); path != nil { |
| 354 | return info, path, exact |
| 355 | } |
| 356 | } |
| 357 | } |
| 358 | return nil, nil, false |
| 359 | } |
| 360 | |
| 361 | // InitialPackages returns a new slice containing the set of initial |
| 362 | // packages (Created + Imported) in unspecified order. |
| 363 | func (prog *Program) InitialPackages() []*PackageInfo { |
| 364 | infos := make([]*PackageInfo, 0, len(prog.Created)+len(prog.Imported)) |
| 365 | infos = append(infos, prog.Created...) |
| 366 | for _, info := range prog.Imported { |
| 367 | infos = append(infos, info) |
| 368 | } |
| 369 | return infos |
| 370 | } |
| 371 | |
| 372 | // Package returns the ASTs and results of type checking for the |
| 373 | // specified package. |
| 374 | func (prog *Program) Package(path string) *PackageInfo { |
| 375 | if info, ok := prog.AllPackages[prog.importMap[path]]; ok { |
| 376 | return info |
| 377 | } |
| 378 | for _, info := range prog.Created { |
| 379 | if path == info.Pkg.Path() { |
| 380 | return info |
| 381 | } |
| 382 | } |
| 383 | return nil |
| 384 | } |
| 385 | |
| 386 | // ---------- Implementation ---------- |
| 387 | |
| 388 | // importer holds the working state of the algorithm. |
| 389 | type importer struct { |
| 390 | conf *Config // the client configuration |
| 391 | start time.Time // for logging |
| 392 | |
| 393 | progMu sync.Mutex // guards prog |
| 394 | prog *Program // the resulting program |
| 395 | |
| 396 | // findpkg is a memoization of FindPackage. |
| 397 | findpkgMu sync.Mutex // guards findpkg |
| 398 | findpkg map[findpkgKey]*findpkgValue |
| 399 | |
| 400 | importedMu sync.Mutex // guards imported |
| 401 | imported map[string]*importInfo // all imported packages (incl. failures) by import path |
| 402 | |
| 403 | // import dependency graph: graph[x][y] => x imports y |
| 404 | // |
| 405 | // Since non-importable packages cannot be cyclic, we ignore |
| 406 | // their imports, thus we only need the subgraph over importable |
| 407 | // packages. Nodes are identified by their import paths. |
| 408 | graphMu sync.Mutex |
| 409 | graph map[string]map[string]bool |
| 410 | } |
| 411 | |
| 412 | type findpkgKey struct { |
| 413 | importPath string |
| 414 | fromDir string |
| 415 | mode build.ImportMode |
| 416 | } |
| 417 | |
| 418 | type findpkgValue struct { |
| 419 | ready chan struct{} // closed to broadcast readiness |
| 420 | bp *build.Package |
| 421 | err error |
| 422 | } |
| 423 | |
| 424 | // importInfo tracks the success or failure of a single import. |
| 425 | // |
| 426 | // Upon completion, exactly one of info and err is non-nil: |
| 427 | // info on successful creation of a package, err otherwise. |
| 428 | // A successful package may still contain type errors. |
| 429 | type importInfo struct { |
| 430 | path string // import path |
| 431 | info *PackageInfo // results of typechecking (including errors) |
| 432 | complete chan struct{} // closed to broadcast that info is set. |
| 433 | } |
| 434 | |
| 435 | // awaitCompletion blocks until ii is complete, |
| 436 | // i.e. the info field is safe to inspect. |
| 437 | func (ii *importInfo) awaitCompletion() { |
| 438 | <-ii.complete // wait for close |
| 439 | } |
| 440 | |
| 441 | // Complete marks ii as complete. |
| 442 | // Its info and err fields will not be subsequently updated. |
| 443 | func (ii *importInfo) Complete(info *PackageInfo) { |
| 444 | if info == nil { |
| 445 | panic("info == nil") |
| 446 | } |
| 447 | ii.info = info |
| 448 | close(ii.complete) |
| 449 | } |
| 450 | |
| 451 | type importError struct { |
| 452 | path string // import path |
| 453 | err error // reason for failure to create a package |
| 454 | } |
| 455 | |
| 456 | // Load creates the initial packages specified by conf.{Create,Import}Pkgs, |
| 457 | // loading their dependencies packages as needed. |
| 458 | // |
| 459 | // On success, Load returns a Program containing a PackageInfo for |
| 460 | // each package. On failure, it returns an error. |
| 461 | // |
| 462 | // If AllowErrors is true, Load will return a Program even if some |
| 463 | // packages contained I/O, parser or type errors, or if dependencies |
| 464 | // were missing. (Such errors are accessible via PackageInfo.Errors. If |
| 465 | // false, Load will fail if any package had an error. |
| 466 | // |
| 467 | // It is an error if no packages were loaded. |
| 468 | func (conf *Config) Load() (*Program, error) { |
| 469 | // Create a simple default error handler for parse/type errors. |
| 470 | if conf.TypeChecker.Error == nil { |
| 471 | conf.TypeChecker.Error = func(e error) { fmt.Fprintln(os.Stderr, e) } |
| 472 | } |
| 473 | |
| 474 | // Set default working directory for relative package references. |
| 475 | if conf.Cwd == "" { |
| 476 | var err error |
| 477 | conf.Cwd, err = os.Getwd() |
| 478 | if err != nil { |
| 479 | return nil, err |
| 480 | } |
| 481 | } |
| 482 | |
| 483 | // Install default FindPackage hook using go/build logic. |
| 484 | if conf.FindPackage == nil { |
| 485 | conf.FindPackage = (*build.Context).Import |
| 486 | } |
| 487 | |
| 488 | prog := &Program{ |
| 489 | Fset: conf.fset(), |
| 490 | Imported: make(map[string]*PackageInfo), |
| 491 | importMap: make(map[string]*types.Package), |
| 492 | AllPackages: make(map[*types.Package]*PackageInfo), |
| 493 | } |
| 494 | |
| 495 | imp := importer{ |
| 496 | conf: conf, |
| 497 | prog: prog, |
| 498 | findpkg: make(map[findpkgKey]*findpkgValue), |
| 499 | imported: make(map[string]*importInfo), |
| 500 | start: time.Now(), |
| 501 | graph: make(map[string]map[string]bool), |
| 502 | } |
| 503 | |
| 504 | // -- loading proper (concurrent phase) -------------------------------- |
| 505 | |
| 506 | var errpkgs []string // packages that contained errors |
| 507 | |
| 508 | // Load the initially imported packages and their dependencies, |
| 509 | // in parallel. |
| 510 | // No vendor check on packages imported from the command line. |
| 511 | infos, importErrors := imp.importAll("", conf.Cwd, conf.ImportPkgs, ignoreVendor) |
| 512 | for _, ie := range importErrors { |
| 513 | conf.TypeChecker.Error(ie.err) // failed to create package |
| 514 | errpkgs = append(errpkgs, ie.path) |
| 515 | } |
| 516 | for _, info := range infos { |
| 517 | prog.Imported[info.Pkg.Path()] = info |
| 518 | } |
| 519 | |
| 520 | // Augment the designated initial packages by their tests. |
| 521 | // Dependencies are loaded in parallel. |
| 522 | var xtestPkgs []*build.Package |
| 523 | for importPath, augment := range conf.ImportPkgs { |
| 524 | if !augment { |
| 525 | continue |
| 526 | } |
| 527 | |
| 528 | // No vendor check on packages imported from command line. |
| 529 | bp, err := imp.findPackage(importPath, conf.Cwd, ignoreVendor) |
| 530 | if err != nil { |
| 531 | // Package not found, or can't even parse package declaration. |
| 532 | // Already reported by previous loop; ignore it. |
| 533 | continue |
| 534 | } |
| 535 | |
| 536 | // Needs external test package? |
| 537 | if len(bp.XTestGoFiles) > 0 { |
| 538 | xtestPkgs = append(xtestPkgs, bp) |
| 539 | } |
| 540 | |
| 541 | // Consult the cache using the canonical package path. |
| 542 | path := bp.ImportPath |
| 543 | imp.importedMu.Lock() // (unnecessary, we're sequential here) |
| 544 | ii, ok := imp.imported[path] |
| 545 | // Paranoid checks added due to issue #11012. |
| 546 | if !ok { |
| 547 | // Unreachable. |
| 548 | // The previous loop called importAll and thus |
| 549 | // startLoad for each path in ImportPkgs, which |
| 550 | // populates imp.imported[path] with a non-zero value. |
| 551 | panic(fmt.Sprintf("imported[%q] not found", path)) |
| 552 | } |
| 553 | if ii == nil { |
| 554 | // Unreachable. |
| 555 | // The ii values in this loop are the same as in |
| 556 | // the previous loop, which enforced the invariant |
| 557 | // that at least one of ii.err and ii.info is non-nil. |
| 558 | panic(fmt.Sprintf("imported[%q] == nil", path)) |
| 559 | } |
| 560 | if ii.info == nil { |
| 561 | // Unreachable. |
| 562 | // awaitCompletion has the postcondition |
| 563 | // ii.info != nil. |
| 564 | panic(fmt.Sprintf("imported[%q].info = nil", path)) |
| 565 | } |
| 566 | info := ii.info |
| 567 | imp.importedMu.Unlock() |
| 568 | |
| 569 | // Parse the in-package test files. |
| 570 | files, errs := imp.conf.parsePackageFiles(bp, 't') |
| 571 | for _, err := range errs { |
| 572 | info.appendError(err) |
| 573 | } |
| 574 | |
| 575 | // The test files augmenting package P cannot be imported, |
| 576 | // but may import packages that import P, |
| 577 | // so we must disable the cycle check. |
| 578 | imp.addFiles(info, files, false) |
| 579 | } |
| 580 | |
| 581 | createPkg := func(path, dir string, files []*ast.File, errs []error) { |
| 582 | info := imp.newPackageInfo(path, dir) |
| 583 | for _, err := range errs { |
| 584 | info.appendError(err) |
| 585 | } |
| 586 | |
| 587 | // Ad hoc packages are non-importable, |
| 588 | // so no cycle check is needed. |
| 589 | // addFiles loads dependencies in parallel. |
| 590 | imp.addFiles(info, files, false) |
| 591 | prog.Created = append(prog.Created, info) |
| 592 | } |
| 593 | |
| 594 | // Create packages specified by conf.CreatePkgs. |
| 595 | for _, cp := range conf.CreatePkgs { |
| 596 | files, errs := parseFiles(conf.fset(), conf.build(), nil, conf.Cwd, cp.Filenames, conf.ParserMode) |
| 597 | files = append(files, cp.Files...) |
| 598 | |
| 599 | path := cp.Path |
| 600 | if path == "" { |
| 601 | if len(files) > 0 { |
| 602 | path = files[0].Name.Name |
| 603 | } else { |
| 604 | path = "(unnamed)" |
| 605 | } |
| 606 | } |
| 607 | |
| 608 | dir := conf.Cwd |
| 609 | if len(files) > 0 && files[0].Pos().IsValid() { |
| 610 | dir = filepath.Dir(conf.fset().File(files[0].Pos()).Name()) |
| 611 | } |
| 612 | createPkg(path, dir, files, errs) |
| 613 | } |
| 614 | |
| 615 | // Create external test packages. |
| 616 | sort.Sort(byImportPath(xtestPkgs)) |
| 617 | for _, bp := range xtestPkgs { |
| 618 | files, errs := imp.conf.parsePackageFiles(bp, 'x') |
| 619 | createPkg(bp.ImportPath+"_test", bp.Dir, files, errs) |
| 620 | } |
| 621 | |
| 622 | // -- finishing up (sequential) ---------------------------------------- |
| 623 | |
| 624 | if len(prog.Imported)+len(prog.Created) == 0 { |
| 625 | return nil, errors.New("no initial packages were loaded") |
| 626 | } |
| 627 | |
| 628 | // Create infos for indirectly imported packages. |
| 629 | // e.g. incomplete packages without syntax, loaded from export data. |
| 630 | for _, obj := range prog.importMap { |
| 631 | info := prog.AllPackages[obj] |
| 632 | if info == nil { |
| 633 | prog.AllPackages[obj] = &PackageInfo{Pkg: obj, Importable: true} |
| 634 | } else { |
| 635 | // finished |
| 636 | info.checker = nil |
| 637 | info.errorFunc = nil |
| 638 | } |
| 639 | } |
| 640 | |
| 641 | if !conf.AllowErrors { |
| 642 | // Report errors in indirectly imported packages. |
| 643 | for _, info := range prog.AllPackages { |
| 644 | if len(info.Errors) > 0 { |
| 645 | errpkgs = append(errpkgs, info.Pkg.Path()) |
| 646 | } |
| 647 | } |
| 648 | if errpkgs != nil { |
| 649 | var more string |
| 650 | if len(errpkgs) > 3 { |
| 651 | more = fmt.Sprintf(" and %d more", len(errpkgs)-3) |
| 652 | errpkgs = errpkgs[:3] |
| 653 | } |
| 654 | return nil, fmt.Errorf("couldn't load packages due to errors: %s%s", |
| 655 | strings.Join(errpkgs, ", "), more) |
| 656 | } |
| 657 | } |
| 658 | |
| 659 | markErrorFreePackages(prog.AllPackages) |
| 660 | |
| 661 | return prog, nil |
| 662 | } |
| 663 | |
| 664 | type byImportPath []*build.Package |
| 665 | |
| 666 | func (b byImportPath) Len() int { return len(b) } |
| 667 | func (b byImportPath) Less(i, j int) bool { return b[i].ImportPath < b[j].ImportPath } |
| 668 | func (b byImportPath) Swap(i, j int) { b[i], b[j] = b[j], b[i] } |
| 669 | |
| 670 | // markErrorFreePackages sets the TransitivelyErrorFree flag on all |
| 671 | // applicable packages. |
| 672 | func markErrorFreePackages(allPackages map[*types.Package]*PackageInfo) { |
| 673 | // Build the transpose of the import graph. |
| 674 | importedBy := make(map[*types.Package]map[*types.Package]bool) |
| 675 | for P := range allPackages { |
| 676 | for _, Q := range P.Imports() { |
| 677 | clients, ok := importedBy[Q] |
| 678 | if !ok { |
| 679 | clients = make(map[*types.Package]bool) |
| 680 | importedBy[Q] = clients |
| 681 | } |
| 682 | clients[P] = true |
| 683 | } |
| 684 | } |
| 685 | |
| 686 | // Find all packages reachable from some error package. |
| 687 | reachable := make(map[*types.Package]bool) |
| 688 | var visit func(*types.Package) |
| 689 | visit = func(p *types.Package) { |
| 690 | if !reachable[p] { |
| 691 | reachable[p] = true |
| 692 | for q := range importedBy[p] { |
| 693 | visit(q) |
| 694 | } |
| 695 | } |
| 696 | } |
| 697 | for _, info := range allPackages { |
| 698 | if len(info.Errors) > 0 { |
| 699 | visit(info.Pkg) |
| 700 | } |
| 701 | } |
| 702 | |
| 703 | // Mark the others as "transitively error-free". |
| 704 | for _, info := range allPackages { |
| 705 | if !reachable[info.Pkg] { |
| 706 | info.TransitivelyErrorFree = true |
| 707 | } |
| 708 | } |
| 709 | } |
| 710 | |
| 711 | // build returns the effective build context. |
| 712 | func (conf *Config) build() *build.Context { |
| 713 | if conf.Build != nil { |
| 714 | return conf.Build |
| 715 | } |
| 716 | return &build.Default |
| 717 | } |
| 718 | |
| 719 | // parsePackageFiles enumerates the files belonging to package path, |
| 720 | // then loads, parses and returns them, plus a list of I/O or parse |
| 721 | // errors that were encountered. |
| 722 | // |
| 723 | // 'which' indicates which files to include: |
| 724 | // |
| 725 | // 'g': include non-test *.go source files (GoFiles + processed CgoFiles) |
| 726 | // 't': include in-package *_test.go source files (TestGoFiles) |
| 727 | // 'x': include external *_test.go source files. (XTestGoFiles) |
| 728 | func (conf *Config) parsePackageFiles(bp *build.Package, which rune) ([]*ast.File, []error) { |
| 729 | if bp.ImportPath == "unsafe" { |
| 730 | return nil, nil |
| 731 | } |
| 732 | var filenames []string |
| 733 | switch which { |
| 734 | case 'g': |
| 735 | filenames = bp.GoFiles |
| 736 | case 't': |
| 737 | filenames = bp.TestGoFiles |
| 738 | case 'x': |
| 739 | filenames = bp.XTestGoFiles |
| 740 | default: |
| 741 | panic(which) |
| 742 | } |
| 743 | |
| 744 | files, errs := parseFiles(conf.fset(), conf.build(), conf.DisplayPath, bp.Dir, filenames, conf.ParserMode) |
| 745 | |
| 746 | // Preprocess CgoFiles and parse the outputs (sequentially). |
| 747 | if which == 'g' && bp.CgoFiles != nil { |
| 748 | cgofiles, err := cgo.ProcessFiles(bp, conf.fset(), conf.DisplayPath, conf.ParserMode) |
| 749 | if err != nil { |
| 750 | errs = append(errs, err) |
| 751 | } else { |
| 752 | files = append(files, cgofiles...) |
| 753 | } |
| 754 | } |
| 755 | |
| 756 | return files, errs |
| 757 | } |
| 758 | |
| 759 | // doImport imports the package denoted by path. |
| 760 | // It implements the types.Importer signature. |
| 761 | // |
| 762 | // It returns an error if a package could not be created |
| 763 | // (e.g. go/build or parse error), but type errors are reported via |
| 764 | // the types.Config.Error callback (the first of which is also saved |
| 765 | // in the package's PackageInfo). |
| 766 | // |
| 767 | // Idempotent. |
| 768 | func (imp *importer) doImport(from *PackageInfo, to string) (*types.Package, error) { |
| 769 | if to == "C" { |
| 770 | // This should be unreachable, but ad hoc packages are |
| 771 | // not currently subject to cgo preprocessing. |
| 772 | // See https://golang.org/issue/11627. |
| 773 | return nil, fmt.Errorf(`the loader doesn't cgo-process ad hoc packages like %q; see Go issue 11627`, |
| 774 | from.Pkg.Path()) |
| 775 | } |
| 776 | |
| 777 | bp, err := imp.findPackage(to, from.dir, 0) |
| 778 | if err != nil { |
| 779 | return nil, err |
| 780 | } |
| 781 | |
| 782 | // The standard unsafe package is handled specially, |
| 783 | // and has no PackageInfo. |
| 784 | if bp.ImportPath == "unsafe" { |
| 785 | return types.Unsafe, nil |
| 786 | } |
| 787 | |
| 788 | // Look for the package in the cache using its canonical path. |
| 789 | path := bp.ImportPath |
| 790 | imp.importedMu.Lock() |
| 791 | ii := imp.imported[path] |
| 792 | imp.importedMu.Unlock() |
| 793 | if ii == nil { |
| 794 | panic("internal error: unexpected import: " + path) |
| 795 | } |
| 796 | if ii.info != nil { |
| 797 | return ii.info.Pkg, nil |
| 798 | } |
| 799 | |
| 800 | // Import of incomplete package: this indicates a cycle. |
| 801 | fromPath := from.Pkg.Path() |
| 802 | if cycle := imp.findPath(path, fromPath); cycle != nil { |
| 803 | // Normalize cycle: start from alphabetically largest node. |
| 804 | pos, start := -1, "" |
| 805 | for i, s := range cycle { |
| 806 | if pos < 0 || s > start { |
| 807 | pos, start = i, s |
| 808 | } |
| 809 | } |
| 810 | cycle = append(cycle, cycle[:pos]...)[pos:] // rotate cycle to start from largest |
| 811 | cycle = append(cycle, cycle[0]) // add start node to end to show cycliness |
| 812 | return nil, fmt.Errorf("import cycle: %s", strings.Join(cycle, " -> ")) |
| 813 | } |
| 814 | |
| 815 | panic("internal error: import of incomplete (yet acyclic) package: " + fromPath) |
| 816 | } |
| 817 | |
| 818 | // findPackage locates the package denoted by the importPath in the |
| 819 | // specified directory. |
| 820 | func (imp *importer) findPackage(importPath, fromDir string, mode build.ImportMode) (*build.Package, error) { |
| 821 | // We use a non-blocking duplicate-suppressing cache (gopl.io §9.7) |
| 822 | // to avoid holding the lock around FindPackage. |
| 823 | key := findpkgKey{importPath, fromDir, mode} |
| 824 | imp.findpkgMu.Lock() |
| 825 | v, ok := imp.findpkg[key] |
| 826 | if ok { |
| 827 | // cache hit |
| 828 | imp.findpkgMu.Unlock() |
| 829 | |
| 830 | <-v.ready // wait for entry to become ready |
| 831 | } else { |
| 832 | // Cache miss: this goroutine becomes responsible for |
| 833 | // populating the map entry and broadcasting its readiness. |
| 834 | v = &findpkgValue{ready: make(chan struct{})} |
| 835 | imp.findpkg[key] = v |
| 836 | imp.findpkgMu.Unlock() |
| 837 | |
| 838 | ioLimit <- true |
| 839 | v.bp, v.err = imp.conf.FindPackage(imp.conf.build(), importPath, fromDir, mode) |
| 840 | <-ioLimit |
| 841 | |
| 842 | if _, ok := v.err.(*build.NoGoError); ok { |
| 843 | v.err = nil // empty directory is not an error |
| 844 | } |
| 845 | |
| 846 | close(v.ready) // broadcast ready condition |
| 847 | } |
| 848 | return v.bp, v.err |
| 849 | } |
| 850 | |
| 851 | // importAll loads, parses, and type-checks the specified packages in |
| 852 | // parallel and returns their completed importInfos in unspecified order. |
| 853 | // |
| 854 | // fromPath is the package path of the importing package, if it is |
| 855 | // importable, "" otherwise. It is used for cycle detection. |
| 856 | // |
| 857 | // fromDir is the directory containing the import declaration that |
| 858 | // caused these imports. |
| 859 | func (imp *importer) importAll(fromPath, fromDir string, imports map[string]bool, mode build.ImportMode) (infos []*PackageInfo, errors []importError) { |
| 860 | if fromPath != "" { |
| 861 | // We're loading a set of imports. |
| 862 | // |
| 863 | // We must record graph edges from the importing package |
| 864 | // to its dependencies, and check for cycles. |
| 865 | imp.graphMu.Lock() |
| 866 | deps, ok := imp.graph[fromPath] |
| 867 | if !ok { |
| 868 | deps = make(map[string]bool) |
| 869 | imp.graph[fromPath] = deps |
| 870 | } |
| 871 | for importPath := range imports { |
| 872 | deps[importPath] = true |
| 873 | } |
| 874 | imp.graphMu.Unlock() |
| 875 | } |
| 876 | |
| 877 | var pending []*importInfo |
| 878 | for importPath := range imports { |
| 879 | if fromPath != "" { |
| 880 | if cycle := imp.findPath(importPath, fromPath); cycle != nil { |
| 881 | // Cycle-forming import: we must not check it |
| 882 | // since it would deadlock. |
| 883 | if trace { |
| 884 | fmt.Fprintf(os.Stderr, "import cycle: %q\n", cycle) |
| 885 | } |
| 886 | continue |
| 887 | } |
| 888 | } |
| 889 | bp, err := imp.findPackage(importPath, fromDir, mode) |
| 890 | if err != nil { |
| 891 | errors = append(errors, importError{ |
| 892 | path: importPath, |
| 893 | err: err, |
| 894 | }) |
| 895 | continue |
| 896 | } |
| 897 | pending = append(pending, imp.startLoad(bp)) |
| 898 | } |
| 899 | |
| 900 | for _, ii := range pending { |
| 901 | ii.awaitCompletion() |
| 902 | infos = append(infos, ii.info) |
| 903 | } |
| 904 | |
| 905 | return infos, errors |
| 906 | } |
| 907 | |
| 908 | // findPath returns an arbitrary path from 'from' to 'to' in the import |
| 909 | // graph, or nil if there was none. |
| 910 | func (imp *importer) findPath(from, to string) []string { |
| 911 | imp.graphMu.Lock() |
| 912 | defer imp.graphMu.Unlock() |
| 913 | |
| 914 | seen := make(map[string]bool) |
| 915 | var search func(stack []string, importPath string) []string |
| 916 | search = func(stack []string, importPath string) []string { |
| 917 | if !seen[importPath] { |
| 918 | seen[importPath] = true |
| 919 | stack = append(stack, importPath) |
| 920 | if importPath == to { |
| 921 | return stack |
| 922 | } |
| 923 | for x := range imp.graph[importPath] { |
| 924 | if p := search(stack, x); p != nil { |
| 925 | return p |
| 926 | } |
| 927 | } |
| 928 | } |
| 929 | return nil |
| 930 | } |
| 931 | return search(make([]string, 0, 20), from) |
| 932 | } |
| 933 | |
| 934 | // startLoad initiates the loading, parsing and type-checking of the |
| 935 | // specified package and its dependencies, if it has not already begun. |
| 936 | // |
| 937 | // It returns an importInfo, not necessarily in a completed state. The |
| 938 | // caller must call awaitCompletion() before accessing its info field. |
| 939 | // |
| 940 | // startLoad is concurrency-safe and idempotent. |
| 941 | func (imp *importer) startLoad(bp *build.Package) *importInfo { |
| 942 | path := bp.ImportPath |
| 943 | imp.importedMu.Lock() |
| 944 | ii, ok := imp.imported[path] |
| 945 | if !ok { |
| 946 | ii = &importInfo{path: path, complete: make(chan struct{})} |
| 947 | imp.imported[path] = ii |
| 948 | go func() { |
| 949 | info := imp.load(bp) |
| 950 | ii.Complete(info) |
| 951 | }() |
| 952 | } |
| 953 | imp.importedMu.Unlock() |
| 954 | |
| 955 | return ii |
| 956 | } |
| 957 | |
| 958 | // load implements package loading by parsing Go source files |
| 959 | // located by go/build. |
| 960 | func (imp *importer) load(bp *build.Package) *PackageInfo { |
| 961 | info := imp.newPackageInfo(bp.ImportPath, bp.Dir) |
| 962 | info.Importable = true |
| 963 | files, errs := imp.conf.parsePackageFiles(bp, 'g') |
| 964 | for _, err := range errs { |
| 965 | info.appendError(err) |
| 966 | } |
| 967 | |
| 968 | imp.addFiles(info, files, true) |
| 969 | |
| 970 | imp.progMu.Lock() |
| 971 | imp.prog.importMap[bp.ImportPath] = info.Pkg |
| 972 | imp.progMu.Unlock() |
| 973 | |
| 974 | return info |
| 975 | } |
| 976 | |
| 977 | // addFiles adds and type-checks the specified files to info, loading |
| 978 | // their dependencies if needed. The order of files determines the |
| 979 | // package initialization order. It may be called multiple times on the |
| 980 | // same package. Errors are appended to the info.Errors field. |
| 981 | // |
| 982 | // cycleCheck determines whether the imports within files create |
| 983 | // dependency edges that should be checked for potential cycles. |
| 984 | func (imp *importer) addFiles(info *PackageInfo, files []*ast.File, cycleCheck bool) { |
| 985 | // Ensure the dependencies are loaded, in parallel. |
| 986 | var fromPath string |
| 987 | if cycleCheck { |
| 988 | fromPath = info.Pkg.Path() |
| 989 | } |
| 990 | // TODO(adonovan): opt: make the caller do scanImports. |
| 991 | // Callers with a build.Package can skip it. |
| 992 | imp.importAll(fromPath, info.dir, scanImports(files), 0) |
| 993 | |
| 994 | if trace { |
| 995 | fmt.Fprintf(os.Stderr, "%s: start %q (%d)\n", |
| 996 | time.Since(imp.start), info.Pkg.Path(), len(files)) |
| 997 | } |
| 998 | |
| 999 | // Don't call checker.Files on Unsafe, even with zero files, |
| 1000 | // because it would mutate the package, which is a global. |
| 1001 | if info.Pkg == types.Unsafe { |
| 1002 | if len(files) > 0 { |
| 1003 | panic(`"unsafe" package contains unexpected files`) |
| 1004 | } |
| 1005 | } else { |
| 1006 | // Ignore the returned (first) error since we |
| 1007 | // already collect them all in the PackageInfo. |
| 1008 | info.checker.Files(files) |
| 1009 | info.Files = append(info.Files, files...) |
| 1010 | } |
| 1011 | |
| 1012 | if imp.conf.AfterTypeCheck != nil { |
| 1013 | imp.conf.AfterTypeCheck(info, files) |
| 1014 | } |
| 1015 | |
| 1016 | if trace { |
| 1017 | fmt.Fprintf(os.Stderr, "%s: stop %q\n", |
| 1018 | time.Since(imp.start), info.Pkg.Path()) |
| 1019 | } |
| 1020 | } |
| 1021 | |
| 1022 | func (imp *importer) newPackageInfo(path, dir string) *PackageInfo { |
| 1023 | var pkg *types.Package |
| 1024 | if path == "unsafe" { |
| 1025 | pkg = types.Unsafe |
| 1026 | } else { |
| 1027 | pkg = types.NewPackage(path, "") |
| 1028 | } |
| 1029 | info := &PackageInfo{ |
| 1030 | Pkg: pkg, |
| 1031 | Info: types.Info{ |
| 1032 | Types: make(map[ast.Expr]types.TypeAndValue), |
| 1033 | Defs: make(map[*ast.Ident]types.Object), |
| 1034 | Uses: make(map[*ast.Ident]types.Object), |
| 1035 | Implicits: make(map[ast.Node]types.Object), |
| 1036 | Scopes: make(map[ast.Node]*types.Scope), |
| 1037 | Selections: make(map[*ast.SelectorExpr]*types.Selection), |
| 1038 | }, |
| 1039 | errorFunc: imp.conf.TypeChecker.Error, |
| 1040 | dir: dir, |
| 1041 | } |
| 1042 | typeparams.InitInstanceInfo(&info.Info) |
| 1043 | |
| 1044 | // Copy the types.Config so we can vary it across PackageInfos. |
| 1045 | tc := imp.conf.TypeChecker |
| 1046 | tc.IgnoreFuncBodies = false |
| 1047 | if f := imp.conf.TypeCheckFuncBodies; f != nil { |
| 1048 | tc.IgnoreFuncBodies = !f(path) |
| 1049 | } |
| 1050 | tc.Importer = closure{imp, info} |
| 1051 | tc.Error = info.appendError // appendError wraps the user's Error function |
| 1052 | |
| 1053 | info.checker = types.NewChecker(&tc, imp.conf.fset(), pkg, &info.Info) |
| 1054 | imp.progMu.Lock() |
| 1055 | imp.prog.AllPackages[pkg] = info |
| 1056 | imp.progMu.Unlock() |
| 1057 | return info |
| 1058 | } |
| 1059 | |
| 1060 | type closure struct { |
| 1061 | imp *importer |
| 1062 | info *PackageInfo |
| 1063 | } |
| 1064 | |
| 1065 | func (c closure) Import(to string) (*types.Package, error) { return c.imp.doImport(c.info, to) } |
| 1066 |
Members