From 280abdbb97b0a7f172c084a485eb1f69dad8f8d7 Mon Sep 17 00:00:00 2001 From: Tyler Bui-Palsulich <26876514+tbpg@users.noreply.github.com> Date: Tue, 13 Apr 2021 08:14:07 -0400 Subject: [PATCH] refactor(internal/godocfx): move pkg loading to separate package (#3917) I'd like to be able to reuse the package loading function for a new tool. So, to make it importable, I refactored it into another package. There are no changes to the code beyond exporting necessary identifiers (and, you know, moving the code). --- internal/godocfx/parse.go | 222 ++++++------------------------- internal/godocfx/pkgload/load.go | 173 ++++++++++++++++++++++++ 2 files changed, 217 insertions(+), 178 deletions(-) create mode 100644 internal/godocfx/pkgload/load.go diff --git a/internal/godocfx/parse.go b/internal/godocfx/parse.go index 476d1b296f4..586c210c35b 100644 --- a/internal/godocfx/parse.go +++ b/internal/godocfx/parse.go @@ -25,9 +25,7 @@ package main import ( "bytes" "fmt" - "go/ast" "go/format" - "go/parser" "go/printer" "go/token" "log" @@ -35,10 +33,10 @@ import ( "path/filepath" "regexp" "sort" - "strconv" "strings" goldmarkcodeblock "cloud.google.com/go/internal/godocfx/goldmark-codeblock" + "cloud.google.com/go/internal/godocfx/pkgload" "cloud.google.com/go/third_party/go/doc" "cloud.google.com/go/third_party/pkgsite" "github.com/yuin/goldmark" @@ -127,11 +125,11 @@ type result struct { func parse(glob string, workingDir string, optionalExtraFiles []string, filter []string) (*result, error) { pages := map[string]*page{} - pkgInfos, err := loadPackages(glob, workingDir, filter) + pkgInfos, err := pkgload.Load(glob, workingDir, filter) if err != nil { return nil, err } - module := pkgInfos[0].pkg.Module + module := pkgInfos[0].Pkg.Module // Filter out extra files that don't exist because some modules don't have a // README. @@ -160,65 +158,65 @@ func parse(glob string, workingDir string, optionalExtraFiles []string, filter [ // independently. for _, pi := range pkgInfos { link := newLinker(pi) - topLevelDecls := pkgsite.TopLevelDecls(pi.doc) + topLevelDecls := pkgsite.TopLevelDecls(pi.Doc) pkgItem := &item{ - UID: pi.doc.ImportPath, - Name: pi.doc.ImportPath, - ID: pi.doc.Name, - Summary: toHTML(pi.doc.Doc), + UID: pi.Doc.ImportPath, + Name: pi.Doc.ImportPath, + ID: pi.Doc.Name, + Summary: toHTML(pi.Doc.Doc), Langs: onlyGo, Type: "package", - Examples: processExamples(pi.doc.Examples, pi.fset), - AltLink: "https://pkg.go.dev/" + pi.doc.ImportPath, + Examples: processExamples(pi.Doc.Examples, pi.Fset), + AltLink: "https://pkg.go.dev/" + pi.Doc.ImportPath, } pkgPage := &page{Items: []*item{pkgItem}} - pages[pi.doc.ImportPath] = pkgPage + pages[pi.Doc.ImportPath] = pkgPage - for _, c := range pi.doc.Consts { + for _, c := range pi.Doc.Consts { name := strings.Join(c.Names, ", ") id := strings.Join(c.Names, ",") - uid := pi.doc.ImportPath + "." + id + uid := pi.Doc.ImportPath + "." + id pkgItem.addChild(child(uid)) pkgPage.addItem(&item{ UID: uid, Name: name, ID: id, - Parent: pi.doc.ImportPath, + Parent: pi.Doc.ImportPath, Type: "const", Summary: c.Doc, Langs: onlyGo, - Syntax: syntax{Content: pkgsite.PrintType(pi.fset, c.Decl, link.toURL, topLevelDecls)}, + Syntax: syntax{Content: pkgsite.PrintType(pi.Fset, c.Decl, link.toURL, topLevelDecls)}, }) } - for _, v := range pi.doc.Vars { + for _, v := range pi.Doc.Vars { name := strings.Join(v.Names, ", ") id := strings.Join(v.Names, ",") - uid := pi.doc.ImportPath + "." + id + uid := pi.Doc.ImportPath + "." + id pkgItem.addChild(child(uid)) pkgPage.addItem(&item{ UID: uid, Name: name, ID: id, - Parent: pi.doc.ImportPath, + Parent: pi.Doc.ImportPath, Type: "variable", Summary: v.Doc, Langs: onlyGo, - Syntax: syntax{Content: pkgsite.PrintType(pi.fset, v.Decl, link.toURL, topLevelDecls)}, + Syntax: syntax{Content: pkgsite.PrintType(pi.Fset, v.Decl, link.toURL, topLevelDecls)}, }) } - for _, t := range pi.doc.Types { - uid := pi.doc.ImportPath + "." + t.Name + for _, t := range pi.Doc.Types { + uid := pi.Doc.ImportPath + "." + t.Name pkgItem.addChild(child(uid)) typeItem := &item{ UID: uid, Name: t.Name, ID: t.Name, - Parent: pi.doc.ImportPath, + Parent: pi.Doc.ImportPath, Type: "type", Summary: t.Doc, Langs: onlyGo, - Syntax: syntax{Content: pkgsite.PrintType(pi.fset, t.Decl, link.toURL, topLevelDecls)}, - Examples: processExamples(t.Examples, pi.fset), + Syntax: syntax{Content: pkgsite.PrintType(pi.Fset, t.Decl, link.toURL, topLevelDecls)}, + Examples: processExamples(t.Examples, pi.Fset), } // Note: items are added as page.Children, rather than // typeItem.Children, as a workaround for the DocFX template. @@ -226,7 +224,7 @@ func parse(glob string, workingDir string, optionalExtraFiles []string, filter [ for _, c := range t.Consts { name := strings.Join(c.Names, ", ") id := strings.Join(c.Names, ",") - cUID := pi.doc.ImportPath + "." + id + cUID := pi.Doc.ImportPath + "." + id pkgItem.addChild(child(cUID)) pkgPage.addItem(&item{ UID: cUID, @@ -236,13 +234,13 @@ func parse(glob string, workingDir string, optionalExtraFiles []string, filter [ Type: "const", Summary: c.Doc, Langs: onlyGo, - Syntax: syntax{Content: pkgsite.PrintType(pi.fset, c.Decl, link.toURL, topLevelDecls)}, + Syntax: syntax{Content: pkgsite.PrintType(pi.Fset, c.Decl, link.toURL, topLevelDecls)}, }) } for _, v := range t.Vars { name := strings.Join(v.Names, ", ") id := strings.Join(v.Names, ",") - cUID := pi.doc.ImportPath + "." + id + cUID := pi.Doc.ImportPath + "." + id pkgItem.addChild(child(cUID)) pkgPage.addItem(&item{ UID: cUID, @@ -252,7 +250,7 @@ func parse(glob string, workingDir string, optionalExtraFiles []string, filter [ Type: "variable", Summary: v.Doc, Langs: onlyGo, - Syntax: syntax{Content: pkgsite.PrintType(pi.fset, v.Decl, link.toURL, topLevelDecls)}, + Syntax: syntax{Content: pkgsite.PrintType(pi.Fset, v.Decl, link.toURL, topLevelDecls)}, }) } @@ -267,8 +265,8 @@ func parse(glob string, workingDir string, optionalExtraFiles []string, filter [ Type: "function", Summary: fn.Doc, Langs: onlyGo, - Syntax: syntax{Content: pkgsite.Synopsis(pi.fset, fn.Decl, link.linkify)}, - Examples: processExamples(fn.Examples, pi.fset), + Syntax: syntax{Content: pkgsite.Synopsis(pi.Fset, fn.Decl, link.linkify)}, + Examples: processExamples(fn.Examples, pi.Fset), }) } for _, fn := range t.Methods { @@ -282,24 +280,24 @@ func parse(glob string, workingDir string, optionalExtraFiles []string, filter [ Type: "method", Summary: fn.Doc, Langs: onlyGo, - Syntax: syntax{Content: pkgsite.Synopsis(pi.fset, fn.Decl, link.linkify)}, - Examples: processExamples(fn.Examples, pi.fset), + Syntax: syntax{Content: pkgsite.Synopsis(pi.Fset, fn.Decl, link.linkify)}, + Examples: processExamples(fn.Examples, pi.Fset), }) } } - for _, fn := range pi.doc.Funcs { - uid := pi.doc.ImportPath + "." + fn.Name + for _, fn := range pi.Doc.Funcs { + uid := pi.Doc.ImportPath + "." + fn.Name pkgItem.addChild(child(uid)) pkgPage.addItem(&item{ UID: uid, Name: fmt.Sprintf("func %s\n", fn.Name), ID: fn.Name, - Parent: pi.doc.ImportPath, + Parent: pi.Doc.ImportPath, Type: "function", Summary: fn.Doc, Langs: onlyGo, - Syntax: syntax{Content: pkgsite.Synopsis(pi.fset, fn.Decl, link.linkify)}, - Examples: processExamples(fn.Examples, pi.fset), + Syntax: syntax{Content: pkgsite.Synopsis(pi.Fset, fn.Decl, link.linkify)}, + Examples: processExamples(fn.Examples, pi.Fset), }) } } @@ -327,16 +325,16 @@ type linker struct { sameDomainModules map[string]*packages.Module } -func newLinker(pi pkgInfo) *linker { +func newLinker(pi pkgload.Info) *linker { sameDomainPrefixes := []string{"cloud.google.com/go"} imports := map[string]string{} sameDomainModules := map[string]*packages.Module{} idToAnchor := map[string]map[string]string{} - for path, pkg := range pi.pkg.Imports { + for path, pkg := range pi.Pkg.Imports { name := pkg.Name - if rename := pi.importRenames[path]; rename != "" { + if rename := pi.ImportRenames[path]; rename != "" { name = rename } imports[name] = path @@ -351,14 +349,14 @@ func newLinker(pi pkgInfo) *linker { } } - idToAnchor[""] = buildIDToAnchor(pi.doc) + idToAnchor[""] = buildIDToAnchor(pi.Doc) return &linker{imports: imports, idToAnchor: idToAnchor, sameDomainModules: sameDomainModules} } // nonWordRegex is based on // https://github.com/googleapis/doc-templates/blob/70eba5908e7b9aef5525d0f1f24194ae750f267e/third_party/docfx/templates/devsite/common.js#L27-L30. -var nonWordRegex = regexp.MustCompile("\\W") +var nonWordRegex = regexp.MustCompile(`\W`) func buildIDToAnchor(pkg *doc.Package) map[string]string { idToAnchor := map[string]string{} @@ -521,7 +519,7 @@ func processExamples(exs []*doc.Example, fset *token.FileSet) []example { return result } -func buildTOC(mod string, pis []pkgInfo, extraFiles []extraFile) tableOfContents { +func buildTOC(mod string, pis []pkgload.Info, extraFiles []extraFile) tableOfContents { toc := tableOfContents{} modTOC := &tocItem{ @@ -551,7 +549,7 @@ func buildTOC(mod string, pis []pkgInfo, extraFiles []extraFile) tableOfContents trimmedPkgs := []string{} for _, pi := range pis { - importPath := pi.doc.ImportPath + importPath := pi.Doc.ImportPath if importPath == mod { continue } @@ -591,138 +589,6 @@ func toHTML(s string) string { return mdBuf.String() } -type pkgInfo struct { - pkg *packages.Package - doc *doc.Package - fset *token.FileSet - // importRenames is a map from package path to local name or "". - importRenames map[string]string -} - -func loadPackages(glob, workingDir string, filter []string) ([]pkgInfo, error) { - config := &packages.Config{ - Mode: packages.NeedName | packages.NeedSyntax | packages.NeedTypes | packages.NeedTypesInfo | packages.NeedModule | packages.NeedImports | packages.NeedDeps, - Tests: true, - Dir: workingDir, - } - - allPkgs, err := packages.Load(config, glob) - if err != nil { - return nil, fmt.Errorf("packages.Load: %v", err) - } - packages.PrintErrors(allPkgs) // Don't fail everything because of one package. - - if len(allPkgs) == 0 { - return nil, fmt.Errorf("pattern %q matched 0 packages", glob) - } - - module := allPkgs[0].Module - skippedModules := map[string]struct{}{} - - // First, collect all of the files grouped by package, including test - // packages. - pkgFiles := map[string][]string{} - - idToPkg := map[string]*packages.Package{} - pkgNames := []string{} - for _, pkg := range allPkgs { - // Ignore filtered packages. - if hasPrefix(pkg.PkgPath, filter) { - continue - } - - id := pkg.ID - // See https://pkg.go.dev/golang.org/x/tools/go/packages#Config. - // The uncompiled test package shows up as "foo_test [foo.test]". - if strings.HasSuffix(id, ".test") || - strings.Contains(id, "internal") || - strings.Contains(id, "third_party") || - (strings.Contains(id, " [") && !strings.Contains(id, "_test [")) { - continue - } - if strings.Contains(id, "_test") { - id = id[0:strings.Index(id, "_test [")] - } else if pkg.Module != nil { - idToPkg[pkg.PkgPath] = pkg - pkgNames = append(pkgNames, pkg.PkgPath) - // The test package doesn't have Module set. - if pkg.Module.Path != module.Path { - skippedModules[pkg.Module.Path] = struct{}{} - continue - } - } - for _, f := range pkg.Syntax { - name := pkg.Fset.File(f.Pos()).Name() - if strings.HasSuffix(name, ".go") { - pkgFiles[id] = append(pkgFiles[id], name) - } - } - } - - sort.Strings(pkgNames) - - result := []pkgInfo{} - - for _, pkgPath := range pkgNames { - // Check if pkgPath has prefix of skipped module. - skip := false - for skipModule := range skippedModules { - if strings.HasPrefix(pkgPath, skipModule) { - skip = true - break - } - } - if skip { - continue - } - parsedFiles := []*ast.File{} - fset := token.NewFileSet() - for _, f := range pkgFiles[pkgPath] { - pf, err := parser.ParseFile(fset, f, nil, parser.ParseComments) - if err != nil { - return nil, fmt.Errorf("ParseFile: %v", err) - } - parsedFiles = append(parsedFiles, pf) - } - - // Parse out GoDoc. - docPkg, err := doc.NewFromFiles(fset, parsedFiles, pkgPath) - if err != nil { - return nil, fmt.Errorf("doc.NewFromFiles: %v", err) - } - - // Extra filter in case the file filtering didn't catch everything. - if !strings.HasPrefix(docPkg.ImportPath, module.Path) { - continue - } - - imports := map[string]string{} - for _, f := range parsedFiles { - for _, i := range f.Imports { - name := "" - // i.Name is nil for imports that aren't renamed. - if i.Name != nil { - name = i.Name.Name - } - iPath, err := strconv.Unquote(i.Path.Value) - if err != nil { - return nil, fmt.Errorf("strconv.Unquote: %v", err) - } - imports[iPath] = name - } - } - - result = append(result, pkgInfo{ - pkg: idToPkg[pkgPath], - doc: docPkg, - fset: fset, - importRenames: imports, - }) - } - - return result, nil -} - func hasPrefix(s string, prefixes []string) bool { for _, prefix := range prefixes { if strings.HasPrefix(s, prefix) { diff --git a/internal/godocfx/pkgload/load.go b/internal/godocfx/pkgload/load.go new file mode 100644 index 00000000000..8ffdb9837fb --- /dev/null +++ b/internal/godocfx/pkgload/load.go @@ -0,0 +1,173 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pkgload + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "sort" + "strconv" + "strings" + + "cloud.google.com/go/third_party/go/doc" + "golang.org/x/tools/go/packages" +) + +// Info holds info about a package. +type Info struct { + Pkg *packages.Package + Doc *doc.Package + Fset *token.FileSet + // ImportRenames is a map from package path to local name or "". + ImportRenames map[string]string +} + +// Load parses the given glob and returns info for the matching packages. +// The workingDir is used for module detection. +// Packages that match the filter are ignored. +func Load(glob, workingDir string, filter []string) ([]Info, error) { + config := &packages.Config{ + Mode: packages.NeedName | packages.NeedSyntax | packages.NeedTypes | packages.NeedTypesInfo | packages.NeedModule | packages.NeedImports | packages.NeedDeps, + Tests: true, + Dir: workingDir, + } + + allPkgs, err := packages.Load(config, glob) + if err != nil { + return nil, fmt.Errorf("packages.Load: %v", err) + } + packages.PrintErrors(allPkgs) // Don't fail everything because of one package. + + if len(allPkgs) == 0 { + return nil, fmt.Errorf("pattern %q matched 0 packages", glob) + } + + module := allPkgs[0].Module + skippedModules := map[string]struct{}{} + + // First, collect all of the files grouped by package, including test + // packages. + pkgFiles := map[string][]string{} + + idToPkg := map[string]*packages.Package{} + pkgNames := []string{} + for _, pkg := range allPkgs { + // Ignore filtered packages. + if hasPrefix(pkg.PkgPath, filter) { + continue + } + + id := pkg.ID + // See https://pkg.go.dev/golang.org/x/tools/go/packages#Config. + // The uncompiled test package shows up as "foo_test [foo.test]". + if strings.HasSuffix(id, ".test") || + strings.Contains(id, "internal") || + strings.Contains(id, "third_party") || + (strings.Contains(id, " [") && !strings.Contains(id, "_test [")) { + continue + } + if strings.Contains(id, "_test") { + id = id[0:strings.Index(id, "_test [")] + } else if pkg.Module != nil { + idToPkg[pkg.PkgPath] = pkg + pkgNames = append(pkgNames, pkg.PkgPath) + // The test package doesn't have Module set. + if pkg.Module.Path != module.Path { + skippedModules[pkg.Module.Path] = struct{}{} + continue + } + } + for _, f := range pkg.Syntax { + name := pkg.Fset.File(f.Pos()).Name() + if strings.HasSuffix(name, ".go") { + pkgFiles[id] = append(pkgFiles[id], name) + } + } + } + + sort.Strings(pkgNames) + + result := []Info{} + + for _, pkgPath := range pkgNames { + // Check if pkgPath has prefix of skipped module. + skip := false + for skipModule := range skippedModules { + if strings.HasPrefix(pkgPath, skipModule) { + skip = true + break + } + } + if skip { + continue + } + parsedFiles := []*ast.File{} + fset := token.NewFileSet() + for _, f := range pkgFiles[pkgPath] { + pf, err := parser.ParseFile(fset, f, nil, parser.ParseComments) + if err != nil { + return nil, fmt.Errorf("ParseFile: %v", err) + } + parsedFiles = append(parsedFiles, pf) + } + + // Parse out GoDoc. + docPkg, err := doc.NewFromFiles(fset, parsedFiles, pkgPath) + if err != nil { + return nil, fmt.Errorf("doc.NewFromFiles: %v", err) + } + + // Extra filter in case the file filtering didn't catch everything. + if !strings.HasPrefix(docPkg.ImportPath, module.Path) { + continue + } + + imports := map[string]string{} + for _, f := range parsedFiles { + for _, i := range f.Imports { + name := "" + // i.Name is nil for imports that aren't renamed. + if i.Name != nil { + name = i.Name.Name + } + iPath, err := strconv.Unquote(i.Path.Value) + if err != nil { + return nil, fmt.Errorf("strconv.Unquote: %v", err) + } + imports[iPath] = name + } + } + + result = append(result, Info{ + Pkg: idToPkg[pkgPath], + Doc: docPkg, + Fset: fset, + ImportRenames: imports, + }) + } + + return result, nil +} + +func hasPrefix(s string, prefixes []string) bool { + for _, prefix := range prefixes { + if strings.HasPrefix(s, prefix) { + return true + } + } + return false +}