1 | // Copyright 2018 The Go Authors. All rights reserved. |
---|---|
2 | // Use of this source code is governed by a BSD-style |
3 | // license that can be found in the LICENSE file. |
4 | |
5 | package packages |
6 | |
7 | import ( |
8 | "encoding/json" |
9 | "fmt" |
10 | "go/parser" |
11 | "go/token" |
12 | "os" |
13 | "path/filepath" |
14 | "regexp" |
15 | "sort" |
16 | "strconv" |
17 | "strings" |
18 | |
19 | "golang.org/x/tools/internal/gocommand" |
20 | ) |
21 | |
22 | // processGolistOverlay provides rudimentary support for adding |
23 | // files that don't exist on disk to an overlay. The results can be |
24 | // sometimes incorrect. |
25 | // TODO(matloob): Handle unsupported cases, including the following: |
26 | // - determining the correct package to add given a new import path |
27 | func (state *golistState) processGolistOverlay(response *responseDeduper) (modifiedPkgs, needPkgs []string, err error) { |
28 | havePkgs := make(map[string]string) // importPath -> non-test package ID |
29 | needPkgsSet := make(map[string]bool) |
30 | modifiedPkgsSet := make(map[string]bool) |
31 | |
32 | pkgOfDir := make(map[string][]*Package) |
33 | for _, pkg := range response.dr.Packages { |
34 | // This is an approximation of import path to id. This can be |
35 | // wrong for tests, vendored packages, and a number of other cases. |
36 | havePkgs[pkg.PkgPath] = pkg.ID |
37 | dir, err := commonDir(pkg.GoFiles) |
38 | if err != nil { |
39 | return nil, nil, err |
40 | } |
41 | if dir != "" { |
42 | pkgOfDir[dir] = append(pkgOfDir[dir], pkg) |
43 | } |
44 | } |
45 | |
46 | // If no new imports are added, it is safe to avoid loading any needPkgs. |
47 | // Otherwise, it's hard to tell which package is actually being loaded |
48 | // (due to vendoring) and whether any modified package will show up |
49 | // in the transitive set of dependencies (because new imports are added, |
50 | // potentially modifying the transitive set of dependencies). |
51 | var overlayAddsImports bool |
52 | |
53 | // If both a package and its test package are created by the overlay, we |
54 | // need the real package first. Process all non-test files before test |
55 | // files, and make the whole process deterministic while we're at it. |
56 | var overlayFiles []string |
57 | for opath := range state.cfg.Overlay { |
58 | overlayFiles = append(overlayFiles, opath) |
59 | } |
60 | sort.Slice(overlayFiles, func(i, j int) bool { |
61 | iTest := strings.HasSuffix(overlayFiles[i], "_test.go") |
62 | jTest := strings.HasSuffix(overlayFiles[j], "_test.go") |
63 | if iTest != jTest { |
64 | return !iTest // non-tests are before tests. |
65 | } |
66 | return overlayFiles[i] < overlayFiles[j] |
67 | }) |
68 | for _, opath := range overlayFiles { |
69 | contents := state.cfg.Overlay[opath] |
70 | base := filepath.Base(opath) |
71 | dir := filepath.Dir(opath) |
72 | var pkg *Package // if opath belongs to both a package and its test variant, this will be the test variant |
73 | var testVariantOf *Package // if opath is a test file, this is the package it is testing |
74 | var fileExists bool |
75 | isTestFile := strings.HasSuffix(opath, "_test.go") |
76 | pkgName, ok := extractPackageName(opath, contents) |
77 | if !ok { |
78 | // Don't bother adding a file that doesn't even have a parsable package statement |
79 | // to the overlay. |
80 | continue |
81 | } |
82 | // If all the overlay files belong to a different package, change the |
83 | // package name to that package. |
84 | maybeFixPackageName(pkgName, isTestFile, pkgOfDir[dir]) |
85 | nextPackage: |
86 | for _, p := range response.dr.Packages { |
87 | if pkgName != p.Name && p.ID != "command-line-arguments" { |
88 | continue |
89 | } |
90 | for _, f := range p.GoFiles { |
91 | if !sameFile(filepath.Dir(f), dir) { |
92 | continue |
93 | } |
94 | // Make sure to capture information on the package's test variant, if needed. |
95 | if isTestFile && !hasTestFiles(p) { |
96 | // TODO(matloob): Are there packages other than the 'production' variant |
97 | // of a package that this can match? This shouldn't match the test main package |
98 | // because the file is generated in another directory. |
99 | testVariantOf = p |
100 | continue nextPackage |
101 | } else if !isTestFile && hasTestFiles(p) { |
102 | // We're examining a test variant, but the overlaid file is |
103 | // a non-test file. Because the overlay implementation |
104 | // (currently) only adds a file to one package, skip this |
105 | // package, so that we can add the file to the production |
106 | // variant of the package. (https://golang.org/issue/36857 |
107 | // tracks handling overlays on both the production and test |
108 | // variant of a package). |
109 | continue nextPackage |
110 | } |
111 | if pkg != nil && p != pkg && pkg.PkgPath == p.PkgPath { |
112 | // We have already seen the production version of the |
113 | // for which p is a test variant. |
114 | if hasTestFiles(p) { |
115 | testVariantOf = pkg |
116 | } |
117 | } |
118 | pkg = p |
119 | if filepath.Base(f) == base { |
120 | fileExists = true |
121 | } |
122 | } |
123 | } |
124 | // The overlay could have included an entirely new package or an |
125 | // ad-hoc package. An ad-hoc package is one that we have manually |
126 | // constructed from inadequate `go list` results for a file= query. |
127 | // It will have the ID command-line-arguments. |
128 | if pkg == nil || pkg.ID == "command-line-arguments" { |
129 | // Try to find the module or gopath dir the file is contained in. |
130 | // Then for modules, add the module opath to the beginning. |
131 | pkgPath, ok, err := state.getPkgPath(dir) |
132 | if err != nil { |
133 | return nil, nil, err |
134 | } |
135 | if !ok { |
136 | break |
137 | } |
138 | var forTest string // only set for x tests |
139 | isXTest := strings.HasSuffix(pkgName, "_test") |
140 | if isXTest { |
141 | forTest = pkgPath |
142 | pkgPath += "_test" |
143 | } |
144 | id := pkgPath |
145 | if isTestFile { |
146 | if isXTest { |
147 | id = fmt.Sprintf("%s [%s.test]", pkgPath, forTest) |
148 | } else { |
149 | id = fmt.Sprintf("%s [%s.test]", pkgPath, pkgPath) |
150 | } |
151 | } |
152 | if pkg != nil { |
153 | // TODO(rstambler): We should change the package's path and ID |
154 | // here. The only issue is that this messes with the roots. |
155 | } else { |
156 | // Try to reclaim a package with the same ID, if it exists in the response. |
157 | for _, p := range response.dr.Packages { |
158 | if reclaimPackage(p, id, opath, contents) { |
159 | pkg = p |
160 | break |
161 | } |
162 | } |
163 | // Otherwise, create a new package. |
164 | if pkg == nil { |
165 | pkg = &Package{ |
166 | PkgPath: pkgPath, |
167 | ID: id, |
168 | Name: pkgName, |
169 | Imports: make(map[string]*Package), |
170 | } |
171 | response.addPackage(pkg) |
172 | havePkgs[pkg.PkgPath] = id |
173 | // Add the production package's sources for a test variant. |
174 | if isTestFile && !isXTest && testVariantOf != nil { |
175 | pkg.GoFiles = append(pkg.GoFiles, testVariantOf.GoFiles...) |
176 | pkg.CompiledGoFiles = append(pkg.CompiledGoFiles, testVariantOf.CompiledGoFiles...) |
177 | // Add the package under test and its imports to the test variant. |
178 | pkg.forTest = testVariantOf.PkgPath |
179 | for k, v := range testVariantOf.Imports { |
180 | pkg.Imports[k] = &Package{ID: v.ID} |
181 | } |
182 | } |
183 | if isXTest { |
184 | pkg.forTest = forTest |
185 | } |
186 | } |
187 | } |
188 | } |
189 | if !fileExists { |
190 | pkg.GoFiles = append(pkg.GoFiles, opath) |
191 | // TODO(matloob): Adding the file to CompiledGoFiles can exhibit the wrong behavior |
192 | // if the file will be ignored due to its build tags. |
193 | pkg.CompiledGoFiles = append(pkg.CompiledGoFiles, opath) |
194 | modifiedPkgsSet[pkg.ID] = true |
195 | } |
196 | imports, err := extractImports(opath, contents) |
197 | if err != nil { |
198 | // Let the parser or type checker report errors later. |
199 | continue |
200 | } |
201 | for _, imp := range imports { |
202 | // TODO(rstambler): If the package is an x test and the import has |
203 | // a test variant, make sure to replace it. |
204 | if _, found := pkg.Imports[imp]; found { |
205 | continue |
206 | } |
207 | overlayAddsImports = true |
208 | id, ok := havePkgs[imp] |
209 | if !ok { |
210 | var err error |
211 | id, err = state.resolveImport(dir, imp) |
212 | if err != nil { |
213 | return nil, nil, err |
214 | } |
215 | } |
216 | pkg.Imports[imp] = &Package{ID: id} |
217 | // Add dependencies to the non-test variant version of this package as well. |
218 | if testVariantOf != nil { |
219 | testVariantOf.Imports[imp] = &Package{ID: id} |
220 | } |
221 | } |
222 | } |
223 | |
224 | // toPkgPath guesses the package path given the id. |
225 | toPkgPath := func(sourceDir, id string) (string, error) { |
226 | if i := strings.IndexByte(id, ' '); i >= 0 { |
227 | return state.resolveImport(sourceDir, id[:i]) |
228 | } |
229 | return state.resolveImport(sourceDir, id) |
230 | } |
231 | |
232 | // Now that new packages have been created, do another pass to determine |
233 | // the new set of missing packages. |
234 | for _, pkg := range response.dr.Packages { |
235 | for _, imp := range pkg.Imports { |
236 | if len(pkg.GoFiles) == 0 { |
237 | return nil, nil, fmt.Errorf("cannot resolve imports for package %q with no Go files", pkg.PkgPath) |
238 | } |
239 | pkgPath, err := toPkgPath(filepath.Dir(pkg.GoFiles[0]), imp.ID) |
240 | if err != nil { |
241 | return nil, nil, err |
242 | } |
243 | if _, ok := havePkgs[pkgPath]; !ok { |
244 | needPkgsSet[pkgPath] = true |
245 | } |
246 | } |
247 | } |
248 | |
249 | if overlayAddsImports { |
250 | needPkgs = make([]string, 0, len(needPkgsSet)) |
251 | for pkg := range needPkgsSet { |
252 | needPkgs = append(needPkgs, pkg) |
253 | } |
254 | } |
255 | modifiedPkgs = make([]string, 0, len(modifiedPkgsSet)) |
256 | for pkg := range modifiedPkgsSet { |
257 | modifiedPkgs = append(modifiedPkgs, pkg) |
258 | } |
259 | return modifiedPkgs, needPkgs, err |
260 | } |
261 | |
262 | // resolveImport finds the ID of a package given its import path. |
263 | // In particular, it will find the right vendored copy when in GOPATH mode. |
264 | func (state *golistState) resolveImport(sourceDir, importPath string) (string, error) { |
265 | env, err := state.getEnv() |
266 | if err != nil { |
267 | return "", err |
268 | } |
269 | if env["GOMOD"] != "" { |
270 | return importPath, nil |
271 | } |
272 | |
273 | searchDir := sourceDir |
274 | for { |
275 | vendorDir := filepath.Join(searchDir, "vendor") |
276 | exists, ok := state.vendorDirs[vendorDir] |
277 | if !ok { |
278 | info, err := os.Stat(vendorDir) |
279 | exists = err == nil && info.IsDir() |
280 | state.vendorDirs[vendorDir] = exists |
281 | } |
282 | |
283 | if exists { |
284 | vendoredPath := filepath.Join(vendorDir, importPath) |
285 | if info, err := os.Stat(vendoredPath); err == nil && info.IsDir() { |
286 | // We should probably check for .go files here, but shame on anyone who fools us. |
287 | path, ok, err := state.getPkgPath(vendoredPath) |
288 | if err != nil { |
289 | return "", err |
290 | } |
291 | if ok { |
292 | return path, nil |
293 | } |
294 | } |
295 | } |
296 | |
297 | // We know we've hit the top of the filesystem when we Dir / and get /, |
298 | // or C:\ and get C:\, etc. |
299 | next := filepath.Dir(searchDir) |
300 | if next == searchDir { |
301 | break |
302 | } |
303 | searchDir = next |
304 | } |
305 | return importPath, nil |
306 | } |
307 | |
308 | func hasTestFiles(p *Package) bool { |
309 | for _, f := range p.GoFiles { |
310 | if strings.HasSuffix(f, "_test.go") { |
311 | return true |
312 | } |
313 | } |
314 | return false |
315 | } |
316 | |
317 | // determineRootDirs returns a mapping from absolute directories that could |
318 | // contain code to their corresponding import path prefixes. |
319 | func (state *golistState) determineRootDirs() (map[string]string, error) { |
320 | env, err := state.getEnv() |
321 | if err != nil { |
322 | return nil, err |
323 | } |
324 | if env["GOMOD"] != "" { |
325 | state.rootsOnce.Do(func() { |
326 | state.rootDirs, state.rootDirsError = state.determineRootDirsModules() |
327 | }) |
328 | } else { |
329 | state.rootsOnce.Do(func() { |
330 | state.rootDirs, state.rootDirsError = state.determineRootDirsGOPATH() |
331 | }) |
332 | } |
333 | return state.rootDirs, state.rootDirsError |
334 | } |
335 | |
336 | func (state *golistState) determineRootDirsModules() (map[string]string, error) { |
337 | // List all of the modules--the first will be the directory for the main |
338 | // module. Any replaced modules will also need to be treated as roots. |
339 | // Editing files in the module cache isn't a great idea, so we don't |
340 | // plan to ever support that. |
341 | out, err := state.invokeGo("list", "-m", "-json", "all") |
342 | if err != nil { |
343 | // 'go list all' will fail if we're outside of a module and |
344 | // GO111MODULE=on. Try falling back without 'all'. |
345 | var innerErr error |
346 | out, innerErr = state.invokeGo("list", "-m", "-json") |
347 | if innerErr != nil { |
348 | return nil, err |
349 | } |
350 | } |
351 | roots := map[string]string{} |
352 | modules := map[string]string{} |
353 | var i int |
354 | for dec := json.NewDecoder(out); dec.More(); { |
355 | mod := new(gocommand.ModuleJSON) |
356 | if err := dec.Decode(mod); err != nil { |
357 | return nil, err |
358 | } |
359 | if mod.Dir != "" && mod.Path != "" { |
360 | // This is a valid module; add it to the map. |
361 | absDir, err := filepath.Abs(mod.Dir) |
362 | if err != nil { |
363 | return nil, err |
364 | } |
365 | modules[absDir] = mod.Path |
366 | // The first result is the main module. |
367 | if i == 0 || mod.Replace != nil && mod.Replace.Path != "" { |
368 | roots[absDir] = mod.Path |
369 | } |
370 | } |
371 | i++ |
372 | } |
373 | return roots, nil |
374 | } |
375 | |
376 | func (state *golistState) determineRootDirsGOPATH() (map[string]string, error) { |
377 | m := map[string]string{} |
378 | for _, dir := range filepath.SplitList(state.mustGetEnv()["GOPATH"]) { |
379 | absDir, err := filepath.Abs(dir) |
380 | if err != nil { |
381 | return nil, err |
382 | } |
383 | m[filepath.Join(absDir, "src")] = "" |
384 | } |
385 | return m, nil |
386 | } |
387 | |
388 | func extractImports(filename string, contents []byte) ([]string, error) { |
389 | f, err := parser.ParseFile(token.NewFileSet(), filename, contents, parser.ImportsOnly) // TODO(matloob): reuse fileset? |
390 | if err != nil { |
391 | return nil, err |
392 | } |
393 | var res []string |
394 | for _, imp := range f.Imports { |
395 | quotedPath := imp.Path.Value |
396 | path, err := strconv.Unquote(quotedPath) |
397 | if err != nil { |
398 | return nil, err |
399 | } |
400 | res = append(res, path) |
401 | } |
402 | return res, nil |
403 | } |
404 | |
405 | // reclaimPackage attempts to reuse a package that failed to load in an overlay. |
406 | // |
407 | // If the package has errors and has no Name, GoFiles, or Imports, |
408 | // then it's possible that it doesn't yet exist on disk. |
409 | func reclaimPackage(pkg *Package, id string, filename string, contents []byte) bool { |
410 | // TODO(rstambler): Check the message of the actual error? |
411 | // It differs between $GOPATH and module mode. |
412 | if pkg.ID != id { |
413 | return false |
414 | } |
415 | if len(pkg.Errors) != 1 { |
416 | return false |
417 | } |
418 | if pkg.Name != "" || pkg.ExportFile != "" { |
419 | return false |
420 | } |
421 | if len(pkg.GoFiles) > 0 || len(pkg.CompiledGoFiles) > 0 || len(pkg.OtherFiles) > 0 { |
422 | return false |
423 | } |
424 | if len(pkg.Imports) > 0 { |
425 | return false |
426 | } |
427 | pkgName, ok := extractPackageName(filename, contents) |
428 | if !ok { |
429 | return false |
430 | } |
431 | pkg.Name = pkgName |
432 | pkg.Errors = nil |
433 | return true |
434 | } |
435 | |
436 | func extractPackageName(filename string, contents []byte) (string, bool) { |
437 | // TODO(rstambler): Check the message of the actual error? |
438 | // It differs between $GOPATH and module mode. |
439 | f, err := parser.ParseFile(token.NewFileSet(), filename, contents, parser.PackageClauseOnly) // TODO(matloob): reuse fileset? |
440 | if err != nil { |
441 | return "", false |
442 | } |
443 | return f.Name.Name, true |
444 | } |
445 | |
446 | // commonDir returns the directory that all files are in, "" if files is empty, |
447 | // or an error if they aren't in the same directory. |
448 | func commonDir(files []string) (string, error) { |
449 | seen := make(map[string]bool) |
450 | for _, f := range files { |
451 | seen[filepath.Dir(f)] = true |
452 | } |
453 | if len(seen) > 1 { |
454 | return "", fmt.Errorf("files (%v) are in more than one directory: %v", files, seen) |
455 | } |
456 | for k := range seen { |
457 | // seen has only one element; return it. |
458 | return k, nil |
459 | } |
460 | return "", nil // no files |
461 | } |
462 | |
463 | // It is possible that the files in the disk directory dir have a different package |
464 | // name from newName, which is deduced from the overlays. If they all have a different |
465 | // package name, and they all have the same package name, then that name becomes |
466 | // the package name. |
467 | // It returns true if it changes the package name, false otherwise. |
468 | func maybeFixPackageName(newName string, isTestFile bool, pkgsOfDir []*Package) { |
469 | names := make(map[string]int) |
470 | for _, p := range pkgsOfDir { |
471 | names[p.Name]++ |
472 | } |
473 | if len(names) != 1 { |
474 | // some files are in different packages |
475 | return |
476 | } |
477 | var oldName string |
478 | for k := range names { |
479 | oldName = k |
480 | } |
481 | if newName == oldName { |
482 | return |
483 | } |
484 | // We might have a case where all of the package names in the directory are |
485 | // the same, but the overlay file is for an x test, which belongs to its |
486 | // own package. If the x test does not yet exist on disk, we may not yet |
487 | // have its package name on disk, but we should not rename the packages. |
488 | // |
489 | // We use a heuristic to determine if this file belongs to an x test: |
490 | // The test file should have a package name whose package name has a _test |
491 | // suffix or looks like "newName_test". |
492 | maybeXTest := strings.HasPrefix(oldName+"_test", newName) || strings.HasSuffix(newName, "_test") |
493 | if isTestFile && maybeXTest { |
494 | return |
495 | } |
496 | for _, p := range pkgsOfDir { |
497 | p.Name = newName |
498 | } |
499 | } |
500 | |
501 | // This function is copy-pasted from |
502 | // https://github.com/golang/go/blob/9706f510a5e2754595d716bd64be8375997311fb/src/cmd/go/internal/search/search.go#L360. |
503 | // It should be deleted when we remove support for overlays from go/packages. |
504 | // |
505 | // NOTE: This does not handle any ./... or ./ style queries, as this function |
506 | // doesn't know the working directory. |
507 | // |
508 | // matchPattern(pattern)(name) reports whether |
509 | // name matches pattern. Pattern is a limited glob |
510 | // pattern in which '...' means 'any string' and there |
511 | // is no other special syntax. |
512 | // Unfortunately, there are two special cases. Quoting "go help packages": |
513 | // |
514 | // First, /... at the end of the pattern can match an empty string, |
515 | // so that net/... matches both net and packages in its subdirectories, like net/http. |
516 | // Second, any slash-separated pattern element containing a wildcard never |
517 | // participates in a match of the "vendor" element in the path of a vendored |
518 | // package, so that ./... does not match packages in subdirectories of |
519 | // ./vendor or ./mycode/vendor, but ./vendor/... and ./mycode/vendor/... do. |
520 | // Note, however, that a directory named vendor that itself contains code |
521 | // is not a vendored package: cmd/vendor would be a command named vendor, |
522 | // and the pattern cmd/... matches it. |
523 | func matchPattern(pattern string) func(name string) bool { |
524 | // Convert pattern to regular expression. |
525 | // The strategy for the trailing /... is to nest it in an explicit ? expression. |
526 | // The strategy for the vendor exclusion is to change the unmatchable |
527 | // vendor strings to a disallowed code point (vendorChar) and to use |
528 | // "(anything but that codepoint)*" as the implementation of the ... wildcard. |
529 | // This is a bit complicated but the obvious alternative, |
530 | // namely a hand-written search like in most shell glob matchers, |
531 | // is too easy to make accidentally exponential. |
532 | // Using package regexp guarantees linear-time matching. |
533 | |
534 | const vendorChar = "\x00" |
535 | |
536 | if strings.Contains(pattern, vendorChar) { |
537 | return func(name string) bool { return false } |
538 | } |
539 | |
540 | re := regexp.QuoteMeta(pattern) |
541 | re = replaceVendor(re, vendorChar) |
542 | switch { |
543 | case strings.HasSuffix(re, `/`+vendorChar+`/\.\.\.`): |
544 | re = strings.TrimSuffix(re, `/`+vendorChar+`/\.\.\.`) + `(/vendor|/` + vendorChar + `/\.\.\.)` |
545 | case re == vendorChar+`/\.\.\.`: |
546 | re = `(/vendor|/` + vendorChar + `/\.\.\.)` |
547 | case strings.HasSuffix(re, `/\.\.\.`): |
548 | re = strings.TrimSuffix(re, `/\.\.\.`) + `(/\.\.\.)?` |
549 | } |
550 | re = strings.ReplaceAll(re, `\.\.\.`, `[^`+vendorChar+`]*`) |
551 | |
552 | reg := regexp.MustCompile(`^` + re + `$`) |
553 | |
554 | return func(name string) bool { |
555 | if strings.Contains(name, vendorChar) { |
556 | return false |
557 | } |
558 | return reg.MatchString(replaceVendor(name, vendorChar)) |
559 | } |
560 | } |
561 | |
562 | // replaceVendor returns the result of replacing |
563 | // non-trailing vendor path elements in x with repl. |
564 | func replaceVendor(x, repl string) string { |
565 | if !strings.Contains(x, "vendor") { |
566 | return x |
567 | } |
568 | elem := strings.Split(x, "/") |
569 | for i := 0; i < len(elem)-1; i++ { |
570 | if elem[i] == "vendor" { |
571 | elem[i] = repl |
572 | } |
573 | } |
574 | return strings.Join(elem, "/") |
575 | } |
576 |
Members