Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(internal): auto-run godocfx on new mods #3069

Merged
merged 3 commits into from Oct 22, 2020
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
16 changes: 0 additions & 16 deletions RELEASING.md
Expand Up @@ -88,13 +88,6 @@ the failures have been resolved.
`git push origin $NV`
1. Update [the releases page](https://github.com/googleapis/google-cloud-go/releases)
with the new release, copying the contents of `CHANGES.md`.
1. Run `go get cloud.google.com/go@vX.Y.Z` then wait for the release
to show up on http://pkg.go.dev/cloud.google.com/go (a few minutes).
1. Go to the [doc publishing job](http://go/google-cloud-go-publish-docs) and
trigger the job with the following environment variables:
`MODULE=cloud.google.com,VERSION=vX.Y.Z`.
Replace the version with the value for the module you're
releasing. See [`publish_docs.sh`](/internal/kokoro/publish_docs.sh).

# How to release a submodule

Expand Down Expand Up @@ -131,15 +124,6 @@ To release a submodule:
`git push origin $NV`
1. Update [the releases page](https://github.com/googleapis/google-cloud-go/releases)
with the new release, copying the contents of `datastore/CHANGES.md`.
1. Run `go get cloud.google.com/go/datastore@vX.Y.Z` then wait for the release
to show up on http://pkg.go.dev/cloud.google.com/go/datastore (a few
minutes).
1. Go to the [doc publishing job](http://go/google-cloud-go-publish-docs) and
trigger the job with the following environment variables:
`MODULE=cloud.google.com/go/datastore,VERSION=vX.Y.Z`.
Replace the module path and version with the values for the module you're
releasing. You can leave all of the other fields blank.
See [`publish_docs.sh`](/internal/kokoro/publish_docs.sh).

# Appendix

Expand Down
1 change: 1 addition & 0 deletions internal/godocfx/go.mod
Expand Up @@ -5,6 +5,7 @@ go 1.15
require (
cloud.google.com/go v0.70.0
cloud.google.com/go/bigquery v1.8.0
cloud.google.com/go/datastore v1.1.0
cloud.google.com/go/storage v1.11.0
github.com/kr/pretty v0.2.1 // indirect
golang.org/x/tools v0.0.0-20201021122455-2be66b663cb6
Expand Down
1 change: 0 additions & 1 deletion internal/godocfx/go.sum
Expand Up @@ -283,7 +283,6 @@ google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1m
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20200827165113-ac2560b5e952 h1:y857ZwFJ60XFsJ00vOc7ouVMLOZp7C+7h03pESkILFY=
google.golang.org/genproto v0.0.0-20200827165113-ac2560b5e952/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20201021134325-0d71844de594 h1:JZWUHUjZJojCHxs9ZZLFsnRGKVBXBoOHGxeTSt6OE+Q=
Expand Down
4 changes: 2 additions & 2 deletions internal/godocfx/godocfx_test.go
Expand Up @@ -37,7 +37,7 @@ func TestMain(m *testing.M) {

func TestParse(t *testing.T) {
mod := "cloud.google.com/go/bigquery"
r, err := parse(mod+"/...", []string{"README.md"})
r, err := parse(mod+"/...", ".", []string{"README.md"})
if err != nil {
t.Fatalf("Parse: %v", err)
}
Expand Down Expand Up @@ -103,7 +103,7 @@ func TestGoldens(t *testing.T) {
extraFiles := []string{"README.md"}

testPath := "cloud.google.com/go/storage"
r, err := parse(testPath, extraFiles)
r, err := parse(testPath, ".", extraFiles)
if err != nil {
t.Fatalf("parse: %v", err)
}
Expand Down
108 changes: 108 additions & 0 deletions internal/godocfx/index.go
@@ -0,0 +1,108 @@
// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build go1.15

package main

import (
"bufio"
"context"
"encoding/json"
"log"
"net/http"
"strings"
"time"
)

// indexer gets a limited list of entries from index.golang.org.
type indexer interface {
get(prefix string, since time.Time) (entries []indexEntry, last time.Time, err error)
}

// indexClient is used to access index.golang.org.
type indexClient struct{}

var _ indexer = indexClient{}

// indexEntry represents a line in the output of index.golang.org/index.
type indexEntry struct {
Path string
Version string
Timestamp time.Time
}

// newModules returns the new modules with the given prefix.
//
// newModules uses index.golang.org/index?since=timestamp to find new module
// versions since the given timestamp.
//
// newModules stores the timestamp of the last successful run with tSaver.
func newModules(ctx context.Context, i indexer, tSaver timeSaver, prefix string) ([]indexEntry, error) {
since, err := tSaver.get(ctx)
if err != nil {
return nil, err
}
fiveMinAgo := time.Now().Add(-5 * time.Minute).UTC() // When to stop processing.
entries := []indexEntry{}
log.Printf("Fetching index.golang.org entries since %s", since.Format(time.RFC3339))
count := 0
for {
count++
var cur []indexEntry
cur, since, err = i.get(prefix, since)
if err != nil {
return nil, err
}
entries = append(entries, cur...)
if since.After(fiveMinAgo) {
break
}
}
log.Printf("Parsed %d index.golang.org pages up to %s", count, since.Format(time.RFC3339))
if err := tSaver.put(ctx, since); err != nil {
return nil, err
}

return entries, nil
}

// get fetches a single chronological page of modules from
// index.golang.org/index.
func (indexClient) get(prefix string, since time.Time) ([]indexEntry, time.Time, error) {
entries := []indexEntry{}
sinceString := since.Format(time.RFC3339)
resp, err := http.Get("https://index.golang.org/index?since=" + sinceString)
if err != nil {
return nil, time.Time{}, err
}

s := bufio.NewScanner(resp.Body)
last := time.Time{}
for s.Scan() {
e := indexEntry{}
if err := json.Unmarshal(s.Bytes(), &e); err != nil {
return nil, time.Time{}, err
}
last = e.Timestamp // Always update the last timestamp.
if !strings.HasPrefix(e.Path, prefix) ||
strings.Contains(e.Path, "internal") ||
strings.Contains(e.Path, "third_party") ||
strings.Contains(e.Version, "-") { // Filter out pseudo-versions.
continue
}
entries = append(entries, e)
}
return entries, last, nil
}
65 changes: 65 additions & 0 deletions internal/godocfx/index_test.go
@@ -0,0 +1,65 @@
// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build go1.15

package main

import (
"context"
"testing"
"time"
)

const wantEntries = 5

type fakeIC struct{}

func (f fakeIC) get(prefix string, since time.Time) (entries []indexEntry, last time.Time, err error) {
e := indexEntry{Timestamp: since.Add(24 * time.Hour)}
return []indexEntry{e}, e.Timestamp, nil
}

type fakeTS struct {
getCalled, putCalled bool
}

func (f *fakeTS) get(context.Context) (time.Time, error) {
f.getCalled = true
t := time.Now().Add(-wantEntries * 24 * time.Hour).UTC()
return t, nil
}

func (f *fakeTS) put(context.Context, time.Time) error {
f.putCalled = true
return nil
}

func TestNewModules(t *testing.T) {
ic := fakeIC{}
ts := &fakeTS{}
entries, err := newModules(context.Background(), ic, ts, "cloud.google.com")
if err != nil {
t.Fatalf("newModules got err: %v", err)
}
if got, want := len(entries), wantEntries; got != want {
t.Errorf("newModules got %d entries, want %d", got, want)
}
if !ts.getCalled {
t.Errorf("fakeTS.get was never called")
}
if !ts.putCalled {
t.Errorf("fakeTS.put was never called")
}
}