diff --git a/RELEASING.md b/RELEASING.md index 953d5b61685..f17f8600e70 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -88,13 +88,6 @@ the failures have been resolved. `git push origin $NV` 1. Update [the releases page](https://github.com/googleapis/google-cloud-go/releases) with the new release, copying the contents of `CHANGES.md`. -1. Run `go get cloud.google.com/go@vX.Y.Z` then wait for the release - to show up on http://pkg.go.dev/cloud.google.com/go (a few minutes). -1. Go to the [doc publishing job](http://go/google-cloud-go-publish-docs) and - trigger the job with the following environment variables: - `MODULE=cloud.google.com,VERSION=vX.Y.Z`. - Replace the version with the value for the module you're - releasing. See [`publish_docs.sh`](/internal/kokoro/publish_docs.sh). # How to release a submodule @@ -131,15 +124,6 @@ To release a submodule: `git push origin $NV` 1. Update [the releases page](https://github.com/googleapis/google-cloud-go/releases) with the new release, copying the contents of `datastore/CHANGES.md`. -1. Run `go get cloud.google.com/go/datastore@vX.Y.Z` then wait for the release - to show up on http://pkg.go.dev/cloud.google.com/go/datastore (a few - minutes). -1. Go to the [doc publishing job](http://go/google-cloud-go-publish-docs) and - trigger the job with the following environment variables: - `MODULE=cloud.google.com/go/datastore,VERSION=vX.Y.Z`. - Replace the module path and version with the values for the module you're - releasing. You can leave all of the other fields blank. - See [`publish_docs.sh`](/internal/kokoro/publish_docs.sh). # Appendix diff --git a/internal/godocfx/go.mod b/internal/godocfx/go.mod index 7363d6bacea..fc3d8e2f438 100644 --- a/internal/godocfx/go.mod +++ b/internal/godocfx/go.mod @@ -5,6 +5,7 @@ go 1.15 require ( cloud.google.com/go v0.70.0 cloud.google.com/go/bigquery v1.8.0 + cloud.google.com/go/datastore v1.1.0 cloud.google.com/go/storage v1.11.0 github.com/kr/pretty v0.2.1 // indirect golang.org/x/tools v0.0.0-20201021122455-2be66b663cb6 diff --git a/internal/godocfx/go.sum b/internal/godocfx/go.sum index 027819df21a..1e750a61c5c 100644 --- a/internal/godocfx/go.sum +++ b/internal/godocfx/go.sum @@ -283,7 +283,6 @@ google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1m google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200827165113-ac2560b5e952 h1:y857ZwFJ60XFsJ00vOc7ouVMLOZp7C+7h03pESkILFY= google.golang.org/genproto v0.0.0-20200827165113-ac2560b5e952/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201021134325-0d71844de594 h1:JZWUHUjZJojCHxs9ZZLFsnRGKVBXBoOHGxeTSt6OE+Q= diff --git a/internal/godocfx/godocfx_test.go b/internal/godocfx/godocfx_test.go index 1f056758869..97f199bc987 100644 --- a/internal/godocfx/godocfx_test.go +++ b/internal/godocfx/godocfx_test.go @@ -37,7 +37,7 @@ func TestMain(m *testing.M) { func TestParse(t *testing.T) { mod := "cloud.google.com/go/bigquery" - r, err := parse(mod+"/...", []string{"README.md"}) + r, err := parse(mod+"/...", ".", []string{"README.md"}) if err != nil { t.Fatalf("Parse: %v", err) } @@ -103,7 +103,7 @@ func TestGoldens(t *testing.T) { extraFiles := []string{"README.md"} testPath := "cloud.google.com/go/storage" - r, err := parse(testPath, extraFiles) + r, err := parse(testPath, ".", extraFiles) if err != nil { t.Fatalf("parse: %v", err) } diff --git a/internal/godocfx/index.go b/internal/godocfx/index.go new file mode 100644 index 00000000000..3cc418b8548 --- /dev/null +++ b/internal/godocfx/index.go @@ -0,0 +1,108 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build go1.15 + +package main + +import ( + "bufio" + "context" + "encoding/json" + "log" + "net/http" + "strings" + "time" +) + +// indexer gets a limited list of entries from index.golang.org. +type indexer interface { + get(prefix string, since time.Time) (entries []indexEntry, last time.Time, err error) +} + +// indexClient is used to access index.golang.org. +type indexClient struct{} + +var _ indexer = indexClient{} + +// indexEntry represents a line in the output of index.golang.org/index. +type indexEntry struct { + Path string + Version string + Timestamp time.Time +} + +// newModules returns the new modules with the given prefix. +// +// newModules uses index.golang.org/index?since=timestamp to find new module +// versions since the given timestamp. +// +// newModules stores the timestamp of the last successful run with tSaver. +func newModules(ctx context.Context, i indexer, tSaver timeSaver, prefix string) ([]indexEntry, error) { + since, err := tSaver.get(ctx) + if err != nil { + return nil, err + } + fiveMinAgo := time.Now().Add(-5 * time.Minute).UTC() // When to stop processing. + entries := []indexEntry{} + log.Printf("Fetching index.golang.org entries since %s", since.Format(time.RFC3339)) + count := 0 + for { + count++ + var cur []indexEntry + cur, since, err = i.get(prefix, since) + if err != nil { + return nil, err + } + entries = append(entries, cur...) + if since.After(fiveMinAgo) { + break + } + } + log.Printf("Parsed %d index.golang.org pages up to %s", count, since.Format(time.RFC3339)) + if err := tSaver.put(ctx, since); err != nil { + return nil, err + } + + return entries, nil +} + +// get fetches a single chronological page of modules from +// index.golang.org/index. +func (indexClient) get(prefix string, since time.Time) ([]indexEntry, time.Time, error) { + entries := []indexEntry{} + sinceString := since.Format(time.RFC3339) + resp, err := http.Get("https://index.golang.org/index?since=" + sinceString) + if err != nil { + return nil, time.Time{}, err + } + + s := bufio.NewScanner(resp.Body) + last := time.Time{} + for s.Scan() { + e := indexEntry{} + if err := json.Unmarshal(s.Bytes(), &e); err != nil { + return nil, time.Time{}, err + } + last = e.Timestamp // Always update the last timestamp. + if !strings.HasPrefix(e.Path, prefix) || + strings.Contains(e.Path, "internal") || + strings.Contains(e.Path, "third_party") || + strings.Contains(e.Version, "-") { // Filter out pseudo-versions. + continue + } + entries = append(entries, e) + } + return entries, last, nil +} diff --git a/internal/godocfx/index_test.go b/internal/godocfx/index_test.go new file mode 100644 index 00000000000..35ad813235a --- /dev/null +++ b/internal/godocfx/index_test.go @@ -0,0 +1,65 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build go1.15 + +package main + +import ( + "context" + "testing" + "time" +) + +const wantEntries = 5 + +type fakeIC struct{} + +func (f fakeIC) get(prefix string, since time.Time) (entries []indexEntry, last time.Time, err error) { + e := indexEntry{Timestamp: since.Add(24 * time.Hour)} + return []indexEntry{e}, e.Timestamp, nil +} + +type fakeTS struct { + getCalled, putCalled bool +} + +func (f *fakeTS) get(context.Context) (time.Time, error) { + f.getCalled = true + t := time.Now().Add(-wantEntries * 24 * time.Hour).UTC() + return t, nil +} + +func (f *fakeTS) put(context.Context, time.Time) error { + f.putCalled = true + return nil +} + +func TestNewModules(t *testing.T) { + ic := fakeIC{} + ts := &fakeTS{} + entries, err := newModules(context.Background(), ic, ts, "cloud.google.com") + if err != nil { + t.Fatalf("newModules got err: %v", err) + } + if got, want := len(entries), wantEntries; got != want { + t.Errorf("newModules got %d entries, want %d", got, want) + } + if !ts.getCalled { + t.Errorf("fakeTS.get was never called") + } + if !ts.putCalled { + t.Errorf("fakeTS.put was never called") + } +} diff --git a/internal/godocfx/main.go b/internal/godocfx/main.go index 11fecd3756a..1d88bc906f4 100644 --- a/internal/godocfx/main.go +++ b/internal/godocfx/main.go @@ -20,11 +20,14 @@ Usage: godocfx [flags] path - cd module && godocfx ./... - godocfx cloud.google.com/go/... - godocfx -print cloud.google.com/go/storage/... - godocfx -out custom/output/dir cloud.google.com/go/... - godocfx -rm cloud.google.com/go/... + # New modules with the given prefix. Delete any previous output. + godocfx -rm -project my-project -new-modules cloud.google.com/go + # Process a single module @latest. + godocfx cloud.google.com/go + # Process and print, instead of save. + godocfx -print cloud.google.com/go/storage@latest + # Change output directory. + godocfx -out custom/output/dir cloud.google.com/go See: * https://dotnet.github.io/docfx/spec/metadata_format_spec.html @@ -37,11 +40,14 @@ TODO: package main import ( + "context" "flag" "fmt" "io" + "io/ioutil" "log" "os" + "os/exec" "path/filepath" "strings" "time" @@ -53,37 +59,120 @@ func main() { print := flag.Bool("print", false, "Print instead of save (default false)") rm := flag.Bool("rm", false, "Delete out directory before generating") outDir := flag.String("out", "obj/api", "Output directory (default obj/api)") + projectID := flag.String("project", "", "Project ID to use. Required when using -new-modules.") + newMods := flag.Bool("new-modules", false, "Process all new modules with the given prefix. Uses timestamp in Datastore. Stores results in $out/$mod.") + + log.SetPrefix("[godocfx] ") + flag.Parse() if flag.NArg() != 1 { - log.Fatalf("%s missing required argument: module path", os.Args[0]) + log.Fatalf("%s missing required argument: module path/prefix", os.Args[0]) + } + + mod := flag.Arg(0) + var mods []indexEntry + if *newMods { + if *projectID == "" { + log.Fatal("Must set -project when using -new-modules") + } + var err error + mods, err = newModules(context.Background(), indexClient{}, &dsTimeSaver{projectID: *projectID}, mod) + if err != nil { + log.Fatal(err) + } + } else { + modPath := mod + version := "latest" + if strings.Contains(mod, "@") { + parts := strings.Split(mod, "@") + if len(parts) != 2 { + log.Fatal("module arg expected only one '@'") + } + modPath = parts[0] + version = parts[1] + } + modPath = strings.TrimSuffix(modPath, "/...") // No /... needed. + mods = []indexEntry{ + { + Path: modPath, + Version: version, + }, + } + } + + if *rm { + os.RemoveAll(*outDir) + } + if len(mods) == 0 { + log.Println("No new modules to process") + return + } + // Create a temp module so we can get the exact version asked for. + tempDir, err := ioutil.TempDir("", "godocfx-*") + if err != nil { + log.Fatalf("ioutil.TempDir: %v", err) + } + runCmd(tempDir, "go", "mod", "init", "cloud.google.com/go/lets-build-some-docs") + for _, m := range mods { + log.Printf("Processing %s@%s", m.Path, m.Version) + + path := *outDir + // If we have more than one module, we need a more specific out path. + if len(mods) > 1 { + path = filepath.Join(path, fmt.Sprintf("%s@%s", m.Path, m.Version)) + } + if err := process(m, tempDir, path, *print); err != nil { + log.Printf("Failed to process %v", m) + } + log.Printf("Done with %s@%s", m.Path, m.Version) + } +} + +func runCmd(dir, name string, args ...string) error { + log.Printf("> [%s] %s %s", dir, name, strings.Join(args, " ")) + cmd := exec.Command(name, args...) + cmd.Dir = dir + if err := cmd.Start(); err != nil { + return fmt.Errorf("Start: %v", err) + } + if err := cmd.Wait(); err != nil { + return fmt.Errorf("Wait: %s", err) + } + return nil +} + +func process(mod indexEntry, tempDir, outDir string, print bool) error { + // Be sure to get the module and run the module loader in the tempDir. + if err := runCmd(tempDir, "go", "mod", "tidy"); err != nil { + return err + } + if err := runCmd(tempDir, "go", "get", mod.Path+"@"+mod.Version); err != nil { + return err } optionalExtraFiles := []string{ "README.md", } - r, err := parse(flag.Arg(0), optionalExtraFiles) + r, err := parse(mod.Path+"/...", tempDir, optionalExtraFiles) if err != nil { - log.Fatal(err) + return fmt.Errorf("parse: %v", err) } - if *print { + if print { if err := yaml.NewEncoder(os.Stdout).Encode(r.pages); err != nil { - log.Fatal(err) + return fmt.Errorf("Encode: %v", err) } fmt.Println("----- toc.yaml") if err := yaml.NewEncoder(os.Stdout).Encode(r.toc); err != nil { - log.Fatal(err) + return fmt.Errorf("Encode: %v", err) } - return + return nil } - if *rm { - os.RemoveAll(*outDir) - } - - if err := write(*outDir, r); err != nil { + if err := write(outDir, r); err != nil { log.Fatalf("write: %v", err) } + return nil } func write(outDir string, r *result) error { diff --git a/internal/godocfx/parse.go b/internal/godocfx/parse.go index c0f788966ed..35e8bb699bd 100644 --- a/internal/godocfx/parse.go +++ b/internal/godocfx/parse.go @@ -108,10 +108,10 @@ type result struct { // to packages.Load as-is. // // extraFiles is a list of paths relative to the module root to include. -func parse(glob string, optionalExtraFiles []string) (*result, error) { +func parse(glob string, workingDir string, optionalExtraFiles []string) (*result, error) { pages := map[string]*page{} - pkgInfos, err := loadPackages(glob) + pkgInfos, err := loadPackages(glob, workingDir) if err != nil { return nil, err } @@ -380,10 +380,11 @@ type pkgInfo struct { fset *token.FileSet } -func loadPackages(glob string) ([]pkgInfo, error) { +func loadPackages(glob, workingDir string) ([]pkgInfo, error) { config := &packages.Config{ Mode: packages.NeedName | packages.NeedSyntax | packages.NeedTypes | packages.NeedTypesInfo | packages.NeedModule, Tests: true, + Dir: workingDir, } allPkgs, err := packages.Load(config, glob) @@ -399,8 +400,6 @@ func loadPackages(glob string) ([]pkgInfo, error) { module := allPkgs[0].Module skippedModules := map[string]struct{}{} - log.Printf("Processing %s@%s", module.Path, module.Version) - // First, collect all of the files grouped by package, including test // packages. pkgFiles := map[string][]string{} diff --git a/internal/godocfx/timesaver.go b/internal/godocfx/timesaver.go new file mode 100644 index 00000000000..cf93234bf72 --- /dev/null +++ b/internal/godocfx/timesaver.go @@ -0,0 +1,73 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build go1.15 + +package main + +import ( + "context" + "fmt" + "log" + "time" + + "cloud.google.com/go/datastore" +) + +// timeSaver gets and puts a time.Time value. +type timeSaver interface { + get(context.Context) (time.Time, error) + put(context.Context, time.Time) error +} + +type dsTimeSaver struct { + projectID string + client *datastore.Client + k *datastore.Key +} + +var _ timeSaver = &dsTimeSaver{} + +// indexTimestamp is the time we've processed until, stored in Datastore. +type indexTimestamp struct { + T time.Time +} + +func (ts *dsTimeSaver) get(ctx context.Context) (time.Time, error) { + if ts.client == nil { + var err error + ts.client, err = datastore.NewClient(ctx, ts.projectID) + if err != nil { + return time.Time{}, err + } + ts.k = datastore.NameKey("godocfx-index", "latest", nil) + } + prevLatest := indexTimestamp{} + if err := ts.client.Get(ctx, ts.k, &prevLatest); err != nil { + if err != datastore.ErrNoSuchEntity { + return time.Time{}, fmt.Errorf("Get: %v", err) + } + // Default to 10 days ago. + prevLatest.T = time.Now().Add(-10 * 24 * time.Hour).UTC() + log.Println("Default to", prevLatest.T) + } + return prevLatest.T.UTC(), nil +} + +func (ts *dsTimeSaver) put(ctx context.Context, t time.Time) error { + if _, err := ts.client.Put(ctx, ts.k, &indexTimestamp{T: t}); err != nil { + return fmt.Errorf("Put: %v", err) + } + return nil +} diff --git a/internal/kokoro/publish_docs.sh b/internal/kokoro/publish_docs.sh index f73c8f6652a..4190ac0d630 100755 --- a/internal/kokoro/publish_docs.sh +++ b/internal/kokoro/publish_docs.sh @@ -19,11 +19,6 @@ set -eo pipefail # Display commands being run. set -x -if [ -z "$MODULE" ] ; then - echo "Must set the MODULE environment variables" - exit 1 -fi - python3 -m pip install --upgrade pip # Workaround for six 1.15 incompatibility issue. python3 -m pip install --use-feature=2020-resolver "gcp-docuploader<2019.0.0" @@ -32,23 +27,23 @@ cd github/google-cloud-go/internal/godocfx go install cd - -if [ -z "$VERSION" ] ; then - VERSION="latest" -fi - cd $(mktemp -d) -# Create a module and get the module@version being asked for. -go mod init cloud.google.com/lets/build/some/docs -go get "$MODULE@$VERSION" - +export GOOGLE_APPLICATION_CREDENTIALS=$KOKORO_KEYSTORE_DIR/72523_go_integration_service_account +# Keep GCLOUD_TESTS_GOLANG_PROJECT_ID in sync with continuous.sh. +export GCLOUD_TESTS_GOLANG_PROJECT_ID=dulcet-port-762 # Generate the YAML and a docs.metadata file. -godocfx "$MODULE/..." +godocfx -project $GCLOUD_TESTS_GOLANG_PROJECT_ID -new-modules cloud.google.com/go cd obj/api || exit 4 -python3 -m docuploader upload \ - --staging-bucket docs-staging-v2 \ - --destination-prefix docfx \ - --credentials "$KOKORO_KEYSTORE_DIR/73713_docuploader_service_account" \ - . +for f in $(find obj/api -name docs.metadata); do + d=$(dirname $f) + cd $d + python3 -m docuploader upload \ + --staging-bucket docs-staging-v2 \ + --destination-prefix docfx \ + --credentials "$KOKORO_KEYSTORE_DIR/73713_docuploader_service_account" \ + . + cd - +done