diff --git a/internal/godocfx/index.go b/internal/godocfx/index.go index 3cc418b8548..83ec59dbdde 100644 --- a/internal/godocfx/index.go +++ b/internal/godocfx/index.go @@ -28,7 +28,7 @@ import ( // indexer gets a limited list of entries from index.golang.org. type indexer interface { - get(prefix string, since time.Time) (entries []indexEntry, last time.Time, err error) + get(prefixes []string, since time.Time) (entries []indexEntry, last time.Time, err error) } // indexClient is used to access index.golang.org. @@ -49,7 +49,7 @@ type indexEntry struct { // versions since the given timestamp. // // newModules stores the timestamp of the last successful run with tSaver. -func newModules(ctx context.Context, i indexer, tSaver timeSaver, prefix string) ([]indexEntry, error) { +func newModules(ctx context.Context, i indexer, tSaver timeSaver, prefixes []string) ([]indexEntry, error) { since, err := tSaver.get(ctx) if err != nil { return nil, err @@ -61,7 +61,7 @@ func newModules(ctx context.Context, i indexer, tSaver timeSaver, prefix string) for { count++ var cur []indexEntry - cur, since, err = i.get(prefix, since) + cur, since, err = i.get(prefixes, since) if err != nil { return nil, err } @@ -80,7 +80,7 @@ func newModules(ctx context.Context, i indexer, tSaver timeSaver, prefix string) // get fetches a single chronological page of modules from // index.golang.org/index. -func (indexClient) get(prefix string, since time.Time) ([]indexEntry, time.Time, error) { +func (indexClient) get(prefixes []string, since time.Time) ([]indexEntry, time.Time, error) { entries := []indexEntry{} sinceString := since.Format(time.RFC3339) resp, err := http.Get("https://index.golang.org/index?since=" + sinceString) @@ -96,7 +96,7 @@ func (indexClient) get(prefix string, since time.Time) ([]indexEntry, time.Time, return nil, time.Time{}, err } last = e.Timestamp // Always update the last timestamp. - if !strings.HasPrefix(e.Path, prefix) || + if !hasPrefix(e.Path, prefixes) || strings.Contains(e.Path, "internal") || strings.Contains(e.Path, "third_party") || strings.Contains(e.Version, "-") { // Filter out pseudo-versions. diff --git a/internal/godocfx/index_test.go b/internal/godocfx/index_test.go index 35ad813235a..db1e0b3ef97 100644 --- a/internal/godocfx/index_test.go +++ b/internal/godocfx/index_test.go @@ -26,7 +26,7 @@ const wantEntries = 5 type fakeIC struct{} -func (f fakeIC) get(prefix string, since time.Time) (entries []indexEntry, last time.Time, err error) { +func (f fakeIC) get(prefixes []string, since time.Time) (entries []indexEntry, last time.Time, err error) { e := indexEntry{Timestamp: since.Add(24 * time.Hour)} return []indexEntry{e}, e.Timestamp, nil } @@ -49,7 +49,7 @@ func (f *fakeTS) put(context.Context, time.Time) error { func TestNewModules(t *testing.T) { ic := fakeIC{} ts := &fakeTS{} - entries, err := newModules(context.Background(), ic, ts, "cloud.google.com") + entries, err := newModules(context.Background(), ic, ts, []string{"cloud.google.com"}) if err != nil { t.Fatalf("newModules got err: %v", err) } diff --git a/internal/godocfx/main.go b/internal/godocfx/main.go index b9fc2aa2565..73cf4092e56 100644 --- a/internal/godocfx/main.go +++ b/internal/godocfx/main.go @@ -58,42 +58,43 @@ func main() { outDir := flag.String("out", "obj/api", "Output directory (default obj/api)") projectID := flag.String("project", "", "Project ID to use. Required when using -new-modules.") newMods := flag.Bool("new-modules", false, "Process all new modules with the given prefix. Uses timestamp in Datastore. Stores results in $out/$mod.") + // TODO: flag to set output URL path log.SetPrefix("[godocfx] ") flag.Parse() - if flag.NArg() != 1 { + if flag.NArg() == 0 { log.Fatalf("%s missing required argument: module path/prefix", os.Args[0]) } - mod := flag.Arg(0) + modNames := flag.Args() var mods []indexEntry if *newMods { if *projectID == "" { log.Fatal("Must set -project when using -new-modules") } var err error - mods, err = newModules(context.Background(), indexClient{}, &dsTimeSaver{projectID: *projectID}, mod) + mods, err = newModules(context.Background(), indexClient{}, &dsTimeSaver{projectID: *projectID}, modNames) if err != nil { log.Fatal(err) } } else { - modPath := mod - version := "latest" - if strings.Contains(mod, "@") { - parts := strings.Split(mod, "@") - if len(parts) != 2 { - log.Fatal("module arg expected only one '@'") + for _, mod := range modNames { + modPath := mod + version := "latest" + if strings.Contains(mod, "@") { + parts := strings.Split(mod, "@") + if len(parts) != 2 { + log.Fatal("module arg expected only one '@'") + } + modPath = parts[0] + version = parts[1] } - modPath = parts[0] - version = parts[1] - } - modPath = strings.TrimSuffix(modPath, "/...") // No /... needed. - mods = []indexEntry{ - { + modPath = strings.TrimSuffix(modPath, "/...") // No /... needed. + mods = append(mods, indexEntry{ Path: modPath, Version: version, - }, + }) } } @@ -105,21 +106,19 @@ func main() { return } // Create a temp module so we can get the exact version asked for. - tempDir, err := ioutil.TempDir("", "godocfx-*") + workingDir, err := ioutil.TempDir("", "godocfx-*") if err != nil { log.Fatalf("ioutil.TempDir: %v", err) } - runCmd(tempDir, "go", "mod", "init", "cloud.google.com/go/lets-build-some-docs") + // Use a fake module that doesn't start with cloud.google.com/go. + runCmd(workingDir, "go", "mod", "init", "cloud.google.com/lets-build-some-docs") for _, m := range mods { log.Printf("Processing %s@%s", m.Path, m.Version) - path := *outDir - // If we have more than one module, we need a more specific out path. - if len(mods) > 1 { - path = filepath.Join(path, fmt.Sprintf("%s@%s", m.Path, m.Version)) - } - if err := process(m, tempDir, path, *print); err != nil { - log.Printf("Failed to process %v", m) + // Always output to specific directory. + path := filepath.Join(*outDir, fmt.Sprintf("%s@%s", m.Path, m.Version)) + if err := process(m, workingDir, path, *print); err != nil { + log.Printf("Failed to process %v: %v", m, err) } log.Printf("Done with %s@%s", m.Path, m.Version) } @@ -138,22 +137,23 @@ func runCmd(dir, name string, args ...string) error { return nil } -func process(mod indexEntry, tempDir, outDir string, print bool) error { +func process(mod indexEntry, workingDir, outDir string, print bool) error { // Be sure to get the module and run the module loader in the tempDir. - if err := runCmd(tempDir, "go", "mod", "tidy"); err != nil { - return err + if err := runCmd(workingDir, "go", "mod", "tidy"); err != nil { + return fmt.Errorf("go mod tidy error: %v", err) } - if err := runCmd(tempDir, "go", "get", mod.Path+"@"+mod.Version); err != nil { - return err + if err := runCmd(workingDir, "go", "get", "-d", "-t", mod.Path+"/...@"+mod.Version); err != nil { + return fmt.Errorf("go get %s@%s: %v", mod.Path, mod.Version, err) } + log.Println("Starting to parse") optionalExtraFiles := []string{} filter := []string{ "cloud.google.com/go/analytics", "cloud.google.com/go/area120", "cloud.google.com/go/gsuiteaddons", } - r, err := parse(mod.Path+"/...", tempDir, optionalExtraFiles, filter) + r, err := parse(mod.Path+"/...", workingDir, optionalExtraFiles, filter) if err != nil { return fmt.Errorf("parse: %v", err) } diff --git a/internal/godocfx/parse.go b/internal/godocfx/parse.go index 586c210c35b..e46677c2c68 100644 --- a/internal/godocfx/parse.go +++ b/internal/godocfx/parse.go @@ -452,8 +452,6 @@ func (l *linker) linkify(s string) string { return fmt.Sprintf("%s%s.%s", prefix, href(l.toURL(pkgPath, ""), pkg), href(l.toURL(pkgPath, name), name)) } -// TODO: link to the right baseURL, with the right module name and version -// pattern. func (l *linker) toURL(pkg, name string) string { if pkg == "" { if anchor := l.idToAnchor[""][name]; anchor != "" { @@ -467,6 +465,7 @@ func (l *linker) toURL(pkg, name string) string { pkgRemainder = pkg[len(mod.Path)+1:] // +1 to skip slash. } // Note: we always link to latest. One day, we'll link to mod.Version. + // Also, other packages may have different paths. baseURL := fmt.Sprintf("/go/docs/reference/%v/latest/%v", mod.Path, pkgRemainder) if anchor := l.idToAnchor[pkg][name]; anchor != "" { return fmt.Sprintf("%s#%s", baseURL, anchor)