Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
feat(docfx): add support for examples (#2884)
Examples are located in _test.go files, which go/packages parses into a
separate foo_test package. So, this requires an extra step of collecting
all files from the package, including test packages, then parsing them.
That way, the go/doc package can associate types/functions/packages with
the examples for them (rather than an unassociated list).
  • Loading branch information
tbpg committed Sep 18, 2020
1 parent 80f55c7 commit 0cc0de3
Show file tree
Hide file tree
Showing 2 changed files with 246 additions and 79 deletions.
226 changes: 147 additions & 79 deletions internal/godocfx/parse.go
Expand Up @@ -17,9 +17,14 @@
package main

import (
"bytes"
"fmt"
"go/ast"
"go/doc"
"go/format"
"go/parser"
"go/printer"
"go/token"
"log"
"sort"
"strings"
Expand Down Expand Up @@ -58,17 +63,23 @@ type syntax struct {
Content string `yaml:"content,omitempty"`
}

type example struct {
Content string `yaml:"content,omitempty"`
Name string `yaml:"name,omitempty"`
}

// item represents a DocFX item.
type item struct {
UID string `yaml:"uid"`
Name string `yaml:"name,omitempty"`
ID string `yaml:"id,omitempty"`
Summary string `yaml:"summary,omitempty"`
Parent string `yaml:"parent,omitempty"`
Type string `yaml:"type,omitempty"`
Langs []string `yaml:"langs,omitempty"`
Syntax syntax `yaml:"syntax,omitempty"`
Children []child `yaml:"children,omitempty"`
UID string `yaml:"uid"`
Name string `yaml:"name,omitempty"`
ID string `yaml:"id,omitempty"`
Summary string `yaml:"summary,omitempty"`
Parent string `yaml:"parent,omitempty"`
Type string `yaml:"type,omitempty"`
Langs []string `yaml:"langs,omitempty"`
Syntax syntax `yaml:"syntax,omitempty"`
Examples []example `yaml:"codeexamples,omitempty"`
Children []child `yaml:"children,omitempty"`
}

func (p *page) addItem(i *item) {
Expand Down Expand Up @@ -109,97 +120,116 @@ func parse(glob string) (map[string]*page, tableOfContents, *packages.Module, er

log.Printf("Processing %s@%s", module.Path, module.Version)

// First, collect all of the files grouped by package, including test
// packages.
pkgFiles := map[string][]string{}
for _, pkg := range pkgs {
if pkg == nil || pkg.Module == nil {
id := pkg.ID
// See https://pkg.go.dev/golang.org/x/tools/go/packages#Config.
// The uncompiled test package shows up as "foo_test [foo.test]".
if strings.HasSuffix(id, ".test") ||
strings.Contains(id, "internal") ||
(strings.Contains(id, " [") && !strings.Contains(id, "_test [")) {
continue
}
if pkg.Module.Path != module.Path {
skippedModules[pkg.Module.Path] = struct{}{}
continue
if strings.Contains(id, "_test") {
id = id[0:strings.Index(id, "_test [")]
} else {
// The test package doesn't have Module set.
if pkg.Module.Path != module.Path {
skippedModules[pkg.Module.Path] = struct{}{}
continue
}
}
// Don't generate docs for tests or internal.
switch {
case strings.HasSuffix(pkg.ID, ".test"),
strings.HasSuffix(pkg.ID, ".test]"),
strings.Contains(pkg.ID, "internal"):
continue
for _, f := range pkg.Syntax {
name := pkg.Fset.File(f.Pos()).Name()
if strings.HasSuffix(name, ".go") {
pkgFiles[id] = append(pkgFiles[id], name)
}
}
}

// Collect all .go files.
files := []*ast.File{}
for _, f := range pkg.Syntax {
tf := pkg.Fset.File(f.Pos())
if strings.HasSuffix(tf.Name(), ".go") {
files = append(files, f)
// Once the files are grouped by package, process each package
// independently.
for pkgPath, files := range pkgFiles {
parsedFiles := []*ast.File{}
fset := token.NewFileSet()
for _, f := range files {
pf, err := parser.ParseFile(fset, f, nil, parser.ParseComments)
if err != nil {
return nil, nil, nil, fmt.Errorf("ParseFile: %v", err)
}
parsedFiles = append(parsedFiles, pf)
}

// Parse out GoDoc.
docPkg, err := doc.NewFromFiles(pkg.Fset, files, pkg.PkgPath)
docPkg, err := doc.NewFromFiles(fset, parsedFiles, pkgPath)
if err != nil {
return nil, nil, nil, fmt.Errorf("doc.NewFromFiles: %v", err)
}

toc = append(toc, &tocItem{
UID: pkg.ID,
Name: pkg.PkgPath,
UID: docPkg.ImportPath,
Name: docPkg.ImportPath,
})

pkgItem := &item{
UID: pkg.ID,
Name: pkg.PkgPath,
ID: pkg.Name,
Summary: docPkg.Doc,
Langs: onlyGo,
Type: "package",
UID: docPkg.ImportPath,
Name: docPkg.ImportPath,
ID: docPkg.Name,
Summary: docPkg.Doc,
Langs: onlyGo,
Type: "package",
Examples: processExamples(docPkg.Examples, fset),
}
pkgPage := &page{Items: []*item{pkgItem}}
pages[pkg.PkgPath] = pkgPage
pages[pkgPath] = pkgPage

for _, c := range docPkg.Consts {
name := strings.Join(c.Names, ", ")
id := strings.Join(c.Names, ",")
uid := pkg.PkgPath + "." + id
uid := docPkg.ImportPath + "." + id
pkgItem.addChild(child(uid))
pkgPage.addItem(&item{
UID: uid,
Name: name,
ID: id,
Parent: pkg.PkgPath,
Parent: docPkg.ImportPath,
Type: "const",
Summary: c.Doc,
Langs: onlyGo,
Syntax: syntax{Content: pkgsite.PrintType(pkg.Fset, c.Decl)},
Syntax: syntax{Content: pkgsite.PrintType(fset, c.Decl)},
})
}
for _, v := range docPkg.Vars {
name := strings.Join(v.Names, ", ")
id := strings.Join(v.Names, ",")
uid := pkg.PkgPath + "." + id
uid := docPkg.ImportPath + "." + id
pkgItem.addChild(child(uid))
pkgPage.addItem(&item{
UID: uid,
Name: name,
ID: id,
Parent: pkg.PkgPath,
Parent: docPkg.ImportPath,
Type: "variable",
Summary: v.Doc,
Langs: onlyGo,
Syntax: syntax{Content: pkgsite.PrintType(pkg.Fset, v.Decl)},
Syntax: syntax{Content: pkgsite.PrintType(fset, v.Decl)},
})
}
for _, t := range docPkg.Types {
uid := pkg.PkgPath + "." + t.Name
uid := docPkg.ImportPath + "." + t.Name
pkgItem.addChild(child(uid))
typeItem := &item{
UID: uid,
Name: t.Name,
ID: t.Name,
Parent: pkg.PkgPath,
Type: "type",
Summary: t.Doc,
Langs: onlyGo,
Syntax: syntax{Content: pkgsite.PrintType(pkg.Fset, t.Decl)},
UID: uid,
Name: t.Name,
ID: t.Name,
Parent: docPkg.ImportPath,
Type: "type",
Summary: t.Doc,
Langs: onlyGo,
Syntax: syntax{Content: pkgsite.PrintType(fset, t.Decl)},
Examples: processExamples(t.Examples, fset),
}
// TODO: items are added as page.Children, rather than
// typeItem.Children, as a workaround for the DocFX template.
Expand All @@ -208,7 +238,7 @@ func parse(glob string) (map[string]*page, tableOfContents, *packages.Module, er
for _, c := range t.Consts {
name := strings.Join(c.Names, ", ")
id := strings.Join(c.Names, ",")
cUID := pkg.PkgPath + "." + id
cUID := docPkg.ImportPath + "." + id
pkgItem.addChild(child(cUID))
pkgPage.addItem(&item{
UID: cUID,
Expand All @@ -218,13 +248,13 @@ func parse(glob string) (map[string]*page, tableOfContents, *packages.Module, er
Type: "const",
Summary: c.Doc,
Langs: onlyGo,
Syntax: syntax{Content: pkgsite.PrintType(pkg.Fset, c.Decl)},
Syntax: syntax{Content: pkgsite.PrintType(fset, c.Decl)},
})
}
for _, v := range t.Vars {
name := strings.Join(v.Names, ", ")
id := strings.Join(v.Names, ",")
cUID := pkg.PkgPath + "." + id
cUID := docPkg.ImportPath + "." + id
pkgItem.addChild(child(cUID))
pkgPage.addItem(&item{
UID: cUID,
Expand All @@ -234,51 +264,54 @@ func parse(glob string) (map[string]*page, tableOfContents, *packages.Module, er
Type: "variable",
Summary: v.Doc,
Langs: onlyGo,
Syntax: syntax{Content: pkgsite.PrintType(pkg.Fset, v.Decl)},
Syntax: syntax{Content: pkgsite.PrintType(fset, v.Decl)},
})
}

for _, fn := range t.Funcs {
fnUID := uid + "." + fn.Name
pkgItem.addChild(child(fnUID))
pkgPage.addItem(&item{
UID: fnUID,
Name: fmt.Sprintf("func %s\n", fn.Name),
ID: fn.Name,
Parent: uid,
Type: "function",
Summary: fn.Doc,
Langs: onlyGo,
Syntax: syntax{Content: pkgsite.Synopsis(pkg.Fset, fn.Decl)},
UID: fnUID,
Name: fmt.Sprintf("func %s\n", fn.Name),
ID: fn.Name,
Parent: uid,
Type: "function",
Summary: fn.Doc,
Langs: onlyGo,
Syntax: syntax{Content: pkgsite.Synopsis(fset, fn.Decl)},
Examples: processExamples(fn.Examples, fset),
})
}
for _, fn := range t.Methods {
fnUID := uid + "." + fn.Name
pkgItem.addChild(child(fnUID))
pkgPage.addItem(&item{
UID: fnUID,
Name: fmt.Sprintf("func (%s) %s\n", fn.Recv, fn.Name),
ID: fn.Name,
Parent: uid,
Type: "function", // Note: this is actually a method.
Summary: fn.Doc,
Langs: onlyGo,
Syntax: syntax{Content: pkgsite.Synopsis(pkg.Fset, fn.Decl)},
UID: fnUID,
Name: fmt.Sprintf("func (%s) %s\n", fn.Recv, fn.Name),
ID: fn.Name,
Parent: uid,
Type: "function", // Note: this is actually a method.
Summary: fn.Doc,
Langs: onlyGo,
Syntax: syntax{Content: pkgsite.Synopsis(fset, fn.Decl)},
Examples: processExamples(fn.Examples, fset),
})
}
}
for _, fn := range docPkg.Funcs {
uid := pkg.PkgPath + "." + fn.Name
uid := docPkg.ImportPath + "." + fn.Name
pkgItem.addChild(child(uid))
pkgPage.addItem(&item{
UID: uid,
Name: fmt.Sprintf("func %s\n", fn.Name),
ID: fn.Name,
Parent: pkg.PkgPath,
Type: "function",
Summary: fn.Doc,
Langs: onlyGo,
Syntax: syntax{Content: pkgsite.Synopsis(pkg.Fset, fn.Decl)},
UID: uid,
Name: fmt.Sprintf("func %s\n", fn.Name),
ID: fn.Name,
Parent: docPkg.ImportPath,
Type: "function",
Summary: fn.Doc,
Langs: onlyGo,
Syntax: syntax{Content: pkgsite.Synopsis(fset, fn.Decl)},
Examples: processExamples(fn.Examples, fset),
})
}
}
Expand All @@ -292,3 +325,38 @@ func parse(glob string) (map[string]*page, tableOfContents, *packages.Module, er
}
return pages, toc, module, nil
}

// processExamples converts the examples to []example.
//
// Surrounding braces and indentation is removed.
func processExamples(exs []*doc.Example, fset *token.FileSet) []example {
result := []example{}
for _, ex := range exs {
buf := &bytes.Buffer{}
var node interface{} = &printer.CommentedNode{
Node: ex.Code,
Comments: ex.Comments,
}
if ex.Play != nil {
node = ex.Play
}
if err := format.Node(buf, fset, node); err != nil {
log.Fatal(err)
}
s := buf.String()
if strings.HasPrefix(s, "{\n") && strings.HasSuffix(s, "\n}") {
lines := strings.Split(s, "\n")
builder := strings.Builder{}
for _, line := range lines[1 : len(lines)-1] {
builder.WriteString(strings.TrimPrefix(line, "\t"))
builder.WriteString("\n")
}
s = builder.String()
}
result = append(result, example{
Content: s,
Name: ex.Suffix,
})
}
return result
}

0 comments on commit 0cc0de3

Please sign in to comment.