From 49f497eab80ce34dfb4ca41f033a5c0429ff5e42 Mon Sep 17 00:00:00 2001 From: Tyler Bui-Palsulich <26876514+tbpg@users.noreply.github.com> Date: Thu, 22 Oct 2020 11:28:02 -0400 Subject: [PATCH] feat(internal): auto-run godocfx on new mods (#3069) This uses index.golang.org to list all modules since a given time. For example, see https://index.golang.org/index?since=2019-04-10T19:08:52.997264Z. Each page is listed in chronological order and is limited to 2000 entries (as of today). This stores the last successful time in Datastore. If there is no time in Datastore, it defaults to 10 days ago. There is a bit of indirection with interfaces to enable testing. In the future, we should add the ability to regenerate the YAML for the latest version of all modules. cc @julieqiu @katiehockman --- RELEASING.md | 16 ---- internal/godocfx/go.mod | 1 + internal/godocfx/go.sum | 1 - internal/godocfx/godocfx_test.go | 4 +- internal/godocfx/index.go | 108 +++++++++++++++++++++++++++ internal/godocfx/index_test.go | 65 ++++++++++++++++ internal/godocfx/main.go | 123 ++++++++++++++++++++++++++----- internal/godocfx/parse.go | 9 +-- internal/godocfx/timesaver.go | 73 ++++++++++++++++++ internal/kokoro/publish_docs.sh | 33 ++++----- 10 files changed, 373 insertions(+), 60 deletions(-) create mode 100644 internal/godocfx/index.go create mode 100644 internal/godocfx/index_test.go create mode 100644 internal/godocfx/timesaver.go diff --git a/RELEASING.md b/RELEASING.md index 953d5b61685..f17f8600e70 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -88,13 +88,6 @@ the failures have been resolved. `git push origin $NV` 1. Update [the releases page](https://github.com/googleapis/google-cloud-go/releases) with the new release, copying the contents of `CHANGES.md`. -1. Run `go get cloud.google.com/go@vX.Y.Z` then wait for the release - to show up on http://pkg.go.dev/cloud.google.com/go (a few minutes). -1. Go to the [doc publishing job](http://go/google-cloud-go-publish-docs) and - trigger the job with the following environment variables: - `MODULE=cloud.google.com,VERSION=vX.Y.Z`. - Replace the version with the value for the module you're - releasing. See [`publish_docs.sh`](/internal/kokoro/publish_docs.sh). # How to release a submodule @@ -131,15 +124,6 @@ To release a submodule: `git push origin $NV` 1. Update [the releases page](https://github.com/googleapis/google-cloud-go/releases) with the new release, copying the contents of `datastore/CHANGES.md`. -1. Run `go get cloud.google.com/go/datastore@vX.Y.Z` then wait for the release - to show up on http://pkg.go.dev/cloud.google.com/go/datastore (a few - minutes). -1. Go to the [doc publishing job](http://go/google-cloud-go-publish-docs) and - trigger the job with the following environment variables: - `MODULE=cloud.google.com/go/datastore,VERSION=vX.Y.Z`. - Replace the module path and version with the values for the module you're - releasing. You can leave all of the other fields blank. - See [`publish_docs.sh`](/internal/kokoro/publish_docs.sh). # Appendix diff --git a/internal/godocfx/go.mod b/internal/godocfx/go.mod index 7363d6bacea..fc3d8e2f438 100644 --- a/internal/godocfx/go.mod +++ b/internal/godocfx/go.mod @@ -5,6 +5,7 @@ go 1.15 require ( cloud.google.com/go v0.70.0 cloud.google.com/go/bigquery v1.8.0 + cloud.google.com/go/datastore v1.1.0 cloud.google.com/go/storage v1.11.0 github.com/kr/pretty v0.2.1 // indirect golang.org/x/tools v0.0.0-20201021122455-2be66b663cb6 diff --git a/internal/godocfx/go.sum b/internal/godocfx/go.sum index 027819df21a..1e750a61c5c 100644 --- a/internal/godocfx/go.sum +++ b/internal/godocfx/go.sum @@ -283,7 +283,6 @@ google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1m google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200827165113-ac2560b5e952 h1:y857ZwFJ60XFsJ00vOc7ouVMLOZp7C+7h03pESkILFY= google.golang.org/genproto v0.0.0-20200827165113-ac2560b5e952/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201021134325-0d71844de594 h1:JZWUHUjZJojCHxs9ZZLFsnRGKVBXBoOHGxeTSt6OE+Q= diff --git a/internal/godocfx/godocfx_test.go b/internal/godocfx/godocfx_test.go index 1f056758869..97f199bc987 100644 --- a/internal/godocfx/godocfx_test.go +++ b/internal/godocfx/godocfx_test.go @@ -37,7 +37,7 @@ func TestMain(m *testing.M) { func TestParse(t *testing.T) { mod := "cloud.google.com/go/bigquery" - r, err := parse(mod+"/...", []string{"README.md"}) + r, err := parse(mod+"/...", ".", []string{"README.md"}) if err != nil { t.Fatalf("Parse: %v", err) } @@ -103,7 +103,7 @@ func TestGoldens(t *testing.T) { extraFiles := []string{"README.md"} testPath := "cloud.google.com/go/storage" - r, err := parse(testPath, extraFiles) + r, err := parse(testPath, ".", extraFiles) if err != nil { t.Fatalf("parse: %v", err) } diff --git a/internal/godocfx/index.go b/internal/godocfx/index.go new file mode 100644 index 00000000000..3cc418b8548 --- /dev/null +++ b/internal/godocfx/index.go @@ -0,0 +1,108 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build go1.15 + +package main + +import ( + "bufio" + "context" + "encoding/json" + "log" + "net/http" + "strings" + "time" +) + +// indexer gets a limited list of entries from index.golang.org. +type indexer interface { + get(prefix string, since time.Time) (entries []indexEntry, last time.Time, err error) +} + +// indexClient is used to access index.golang.org. +type indexClient struct{} + +var _ indexer = indexClient{} + +// indexEntry represents a line in the output of index.golang.org/index. +type indexEntry struct { + Path string + Version string + Timestamp time.Time +} + +// newModules returns the new modules with the given prefix. +// +// newModules uses index.golang.org/index?since=timestamp to find new module +// versions since the given timestamp. +// +// newModules stores the timestamp of the last successful run with tSaver. +func newModules(ctx context.Context, i indexer, tSaver timeSaver, prefix string) ([]indexEntry, error) { + since, err := tSaver.get(ctx) + if err != nil { + return nil, err + } + fiveMinAgo := time.Now().Add(-5 * time.Minute).UTC() // When to stop processing. + entries := []indexEntry{} + log.Printf("Fetching index.golang.org entries since %s", since.Format(time.RFC3339)) + count := 0 + for { + count++ + var cur []indexEntry + cur, since, err = i.get(prefix, since) + if err != nil { + return nil, err + } + entries = append(entries, cur...) + if since.After(fiveMinAgo) { + break + } + } + log.Printf("Parsed %d index.golang.org pages up to %s", count, since.Format(time.RFC3339)) + if err := tSaver.put(ctx, since); err != nil { + return nil, err + } + + return entries, nil +} + +// get fetches a single chronological page of modules from +// index.golang.org/index. +func (indexClient) get(prefix string, since time.Time) ([]indexEntry, time.Time, error) { + entries := []indexEntry{} + sinceString := since.Format(time.RFC3339) + resp, err := http.Get("https://index.golang.org/index?since=" + sinceString) + if err != nil { + return nil, time.Time{}, err + } + + s := bufio.NewScanner(resp.Body) + last := time.Time{} + for s.Scan() { + e := indexEntry{} + if err := json.Unmarshal(s.Bytes(), &e); err != nil { + return nil, time.Time{}, err + } + last = e.Timestamp // Always update the last timestamp. + if !strings.HasPrefix(e.Path, prefix) || + strings.Contains(e.Path, "internal") || + strings.Contains(e.Path, "third_party") || + strings.Contains(e.Version, "-") { // Filter out pseudo-versions. + continue + } + entries = append(entries, e) + } + return entries, last, nil +} diff --git a/internal/godocfx/index_test.go b/internal/godocfx/index_test.go new file mode 100644 index 00000000000..35ad813235a --- /dev/null +++ b/internal/godocfx/index_test.go @@ -0,0 +1,65 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build go1.15 + +package main + +import ( + "context" + "testing" + "time" +) + +const wantEntries = 5 + +type fakeIC struct{} + +func (f fakeIC) get(prefix string, since time.Time) (entries []indexEntry, last time.Time, err error) { + e := indexEntry{Timestamp: since.Add(24 * time.Hour)} + return []indexEntry{e}, e.Timestamp, nil +} + +type fakeTS struct { + getCalled, putCalled bool +} + +func (f *fakeTS) get(context.Context) (time.Time, error) { + f.getCalled = true + t := time.Now().Add(-wantEntries * 24 * time.Hour).UTC() + return t, nil +} + +func (f *fakeTS) put(context.Context, time.Time) error { + f.putCalled = true + return nil +} + +func TestNewModules(t *testing.T) { + ic := fakeIC{} + ts := &fakeTS{} + entries, err := newModules(context.Background(), ic, ts, "cloud.google.com") + if err != nil { + t.Fatalf("newModules got err: %v", err) + } + if got, want := len(entries), wantEntries; got != want { + t.Errorf("newModules got %d entries, want %d", got, want) + } + if !ts.getCalled { + t.Errorf("fakeTS.get was never called") + } + if !ts.putCalled { + t.Errorf("fakeTS.put was never called") + } +} diff --git a/internal/godocfx/main.go b/internal/godocfx/main.go index 11fecd3756a..1d88bc906f4 100644 --- a/internal/godocfx/main.go +++ b/internal/godocfx/main.go @@ -20,11 +20,14 @@ Usage: godocfx [flags] path - cd module && godocfx ./... - godocfx cloud.google.com/go/... - godocfx -print cloud.google.com/go/storage/... - godocfx -out custom/output/dir cloud.google.com/go/... - godocfx -rm cloud.google.com/go/... + # New modules with the given prefix. Delete any previous output. + godocfx -rm -project my-project -new-modules cloud.google.com/go + # Process a single module @latest. + godocfx cloud.google.com/go + # Process and print, instead of save. + godocfx -print cloud.google.com/go/storage@latest + # Change output directory. + godocfx -out custom/output/dir cloud.google.com/go See: * https://dotnet.github.io/docfx/spec/metadata_format_spec.html @@ -37,11 +40,14 @@ TODO: package main import ( + "context" "flag" "fmt" "io" + "io/ioutil" "log" "os" + "os/exec" "path/filepath" "strings" "time" @@ -53,37 +59,120 @@ func main() { print := flag.Bool("print", false, "Print instead of save (default false)") rm := flag.Bool("rm", false, "Delete out directory before generating") outDir := flag.String("out", "obj/api", "Output directory (default obj/api)") + projectID := flag.String("project", "", "Project ID to use. Required when using -new-modules.") + newMods := flag.Bool("new-modules", false, "Process all new modules with the given prefix. Uses timestamp in Datastore. Stores results in $out/$mod.") + + log.SetPrefix("[godocfx] ") + flag.Parse() if flag.NArg() != 1 { - log.Fatalf("%s missing required argument: module path", os.Args[0]) + log.Fatalf("%s missing required argument: module path/prefix", os.Args[0]) + } + + mod := flag.Arg(0) + var mods []indexEntry + if *newMods { + if *projectID == "" { + log.Fatal("Must set -project when using -new-modules") + } + var err error + mods, err = newModules(context.Background(), indexClient{}, &dsTimeSaver{projectID: *projectID}, mod) + if err != nil { + log.Fatal(err) + } + } else { + modPath := mod + version := "latest" + if strings.Contains(mod, "@") { + parts := strings.Split(mod, "@") + if len(parts) != 2 { + log.Fatal("module arg expected only one '@'") + } + modPath = parts[0] + version = parts[1] + } + modPath = strings.TrimSuffix(modPath, "/...") // No /... needed. + mods = []indexEntry{ + { + Path: modPath, + Version: version, + }, + } + } + + if *rm { + os.RemoveAll(*outDir) + } + if len(mods) == 0 { + log.Println("No new modules to process") + return + } + // Create a temp module so we can get the exact version asked for. + tempDir, err := ioutil.TempDir("", "godocfx-*") + if err != nil { + log.Fatalf("ioutil.TempDir: %v", err) + } + runCmd(tempDir, "go", "mod", "init", "cloud.google.com/go/lets-build-some-docs") + for _, m := range mods { + log.Printf("Processing %s@%s", m.Path, m.Version) + + path := *outDir + // If we have more than one module, we need a more specific out path. + if len(mods) > 1 { + path = filepath.Join(path, fmt.Sprintf("%s@%s", m.Path, m.Version)) + } + if err := process(m, tempDir, path, *print); err != nil { + log.Printf("Failed to process %v", m) + } + log.Printf("Done with %s@%s", m.Path, m.Version) + } +} + +func runCmd(dir, name string, args ...string) error { + log.Printf("> [%s] %s %s", dir, name, strings.Join(args, " ")) + cmd := exec.Command(name, args...) + cmd.Dir = dir + if err := cmd.Start(); err != nil { + return fmt.Errorf("Start: %v", err) + } + if err := cmd.Wait(); err != nil { + return fmt.Errorf("Wait: %s", err) + } + return nil +} + +func process(mod indexEntry, tempDir, outDir string, print bool) error { + // Be sure to get the module and run the module loader in the tempDir. + if err := runCmd(tempDir, "go", "mod", "tidy"); err != nil { + return err + } + if err := runCmd(tempDir, "go", "get", mod.Path+"@"+mod.Version); err != nil { + return err } optionalExtraFiles := []string{ "README.md", } - r, err := parse(flag.Arg(0), optionalExtraFiles) + r, err := parse(mod.Path+"/...", tempDir, optionalExtraFiles) if err != nil { - log.Fatal(err) + return fmt.Errorf("parse: %v", err) } - if *print { + if print { if err := yaml.NewEncoder(os.Stdout).Encode(r.pages); err != nil { - log.Fatal(err) + return fmt.Errorf("Encode: %v", err) } fmt.Println("----- toc.yaml") if err := yaml.NewEncoder(os.Stdout).Encode(r.toc); err != nil { - log.Fatal(err) + return fmt.Errorf("Encode: %v", err) } - return + return nil } - if *rm { - os.RemoveAll(*outDir) - } - - if err := write(*outDir, r); err != nil { + if err := write(outDir, r); err != nil { log.Fatalf("write: %v", err) } + return nil } func write(outDir string, r *result) error { diff --git a/internal/godocfx/parse.go b/internal/godocfx/parse.go index c0f788966ed..35e8bb699bd 100644 --- a/internal/godocfx/parse.go +++ b/internal/godocfx/parse.go @@ -108,10 +108,10 @@ type result struct { // to packages.Load as-is. // // extraFiles is a list of paths relative to the module root to include. -func parse(glob string, optionalExtraFiles []string) (*result, error) { +func parse(glob string, workingDir string, optionalExtraFiles []string) (*result, error) { pages := map[string]*page{} - pkgInfos, err := loadPackages(glob) + pkgInfos, err := loadPackages(glob, workingDir) if err != nil { return nil, err } @@ -380,10 +380,11 @@ type pkgInfo struct { fset *token.FileSet } -func loadPackages(glob string) ([]pkgInfo, error) { +func loadPackages(glob, workingDir string) ([]pkgInfo, error) { config := &packages.Config{ Mode: packages.NeedName | packages.NeedSyntax | packages.NeedTypes | packages.NeedTypesInfo | packages.NeedModule, Tests: true, + Dir: workingDir, } allPkgs, err := packages.Load(config, glob) @@ -399,8 +400,6 @@ func loadPackages(glob string) ([]pkgInfo, error) { module := allPkgs[0].Module skippedModules := map[string]struct{}{} - log.Printf("Processing %s@%s", module.Path, module.Version) - // First, collect all of the files grouped by package, including test // packages. pkgFiles := map[string][]string{} diff --git a/internal/godocfx/timesaver.go b/internal/godocfx/timesaver.go new file mode 100644 index 00000000000..cf93234bf72 --- /dev/null +++ b/internal/godocfx/timesaver.go @@ -0,0 +1,73 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build go1.15 + +package main + +import ( + "context" + "fmt" + "log" + "time" + + "cloud.google.com/go/datastore" +) + +// timeSaver gets and puts a time.Time value. +type timeSaver interface { + get(context.Context) (time.Time, error) + put(context.Context, time.Time) error +} + +type dsTimeSaver struct { + projectID string + client *datastore.Client + k *datastore.Key +} + +var _ timeSaver = &dsTimeSaver{} + +// indexTimestamp is the time we've processed until, stored in Datastore. +type indexTimestamp struct { + T time.Time +} + +func (ts *dsTimeSaver) get(ctx context.Context) (time.Time, error) { + if ts.client == nil { + var err error + ts.client, err = datastore.NewClient(ctx, ts.projectID) + if err != nil { + return time.Time{}, err + } + ts.k = datastore.NameKey("godocfx-index", "latest", nil) + } + prevLatest := indexTimestamp{} + if err := ts.client.Get(ctx, ts.k, &prevLatest); err != nil { + if err != datastore.ErrNoSuchEntity { + return time.Time{}, fmt.Errorf("Get: %v", err) + } + // Default to 10 days ago. + prevLatest.T = time.Now().Add(-10 * 24 * time.Hour).UTC() + log.Println("Default to", prevLatest.T) + } + return prevLatest.T.UTC(), nil +} + +func (ts *dsTimeSaver) put(ctx context.Context, t time.Time) error { + if _, err := ts.client.Put(ctx, ts.k, &indexTimestamp{T: t}); err != nil { + return fmt.Errorf("Put: %v", err) + } + return nil +} diff --git a/internal/kokoro/publish_docs.sh b/internal/kokoro/publish_docs.sh index f73c8f6652a..4190ac0d630 100755 --- a/internal/kokoro/publish_docs.sh +++ b/internal/kokoro/publish_docs.sh @@ -19,11 +19,6 @@ set -eo pipefail # Display commands being run. set -x -if [ -z "$MODULE" ] ; then - echo "Must set the MODULE environment variables" - exit 1 -fi - python3 -m pip install --upgrade pip # Workaround for six 1.15 incompatibility issue. python3 -m pip install --use-feature=2020-resolver "gcp-docuploader<2019.0.0" @@ -32,23 +27,23 @@ cd github/google-cloud-go/internal/godocfx go install cd - -if [ -z "$VERSION" ] ; then - VERSION="latest" -fi - cd $(mktemp -d) -# Create a module and get the module@version being asked for. -go mod init cloud.google.com/lets/build/some/docs -go get "$MODULE@$VERSION" - +export GOOGLE_APPLICATION_CREDENTIALS=$KOKORO_KEYSTORE_DIR/72523_go_integration_service_account +# Keep GCLOUD_TESTS_GOLANG_PROJECT_ID in sync with continuous.sh. +export GCLOUD_TESTS_GOLANG_PROJECT_ID=dulcet-port-762 # Generate the YAML and a docs.metadata file. -godocfx "$MODULE/..." +godocfx -project $GCLOUD_TESTS_GOLANG_PROJECT_ID -new-modules cloud.google.com/go cd obj/api || exit 4 -python3 -m docuploader upload \ - --staging-bucket docs-staging-v2 \ - --destination-prefix docfx \ - --credentials "$KOKORO_KEYSTORE_DIR/73713_docuploader_service_account" \ - . +for f in $(find obj/api -name docs.metadata); do + d=$(dirname $f) + cd $d + python3 -m docuploader upload \ + --staging-bucket docs-staging-v2 \ + --destination-prefix docfx \ + --credentials "$KOKORO_KEYSTORE_DIR/73713_docuploader_service_account" \ + . + cd - +done