Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
feat(internal): auto-generate snippets (#3949)
I created a new package to make it easier to run than calling the binary. I included a binary for local development.

I needed to install build-base in the Docker image to get gcc, which was needed to Load all packages.

I modified gapics.go to replace genproto in every go.mod because the changes need to be visible everywhere to Load every module.

I ran the generator locally and it seems to work. I only included one directory in this PR because there are a lot of files. I can upload the rest in this PR after initial review.

Updates #3916.
  • Loading branch information
tbpg committed Apr 16, 2021
1 parent 8b4adbf commit b70e0fc
Show file tree
Hide file tree
Showing 16 changed files with 802 additions and 49 deletions.
2 changes: 1 addition & 1 deletion internal/gapicgen/cmd/genbot/Dockerfile
Expand Up @@ -5,7 +5,7 @@ ENV GO111MODULE on
RUN apk update && \
apk add ca-certificates wget git unzip
# Install bash and ssh tools (needed to run regen.sh etc).
RUN apk add bash openssh openssh-client
RUN apk add bash openssh openssh-client build-base
RUN which bash

# Install libc compatibility (required by protoc and go)
Expand Down
97 changes: 85 additions & 12 deletions internal/gapicgen/generator/gapics.go
Expand Up @@ -18,11 +18,14 @@ import (
"context"
"encoding/json"
"fmt"
"io/fs"
"log"
"os"
"path/filepath"
"strings"

"cloud.google.com/go/internal/gensnippets"
"golang.org/x/sys/execabs"
"gopkg.in/yaml.v2"
)

Expand Down Expand Up @@ -74,7 +77,18 @@ func (g *GapicGenerator) Regen(ctx context.Context) error {
return err
}

if err := g.addModReplaceGenproto(); err != nil {
if err := forEachMod(g.googleCloudDir, g.addModReplaceGenproto); err != nil {
return err
}

snippetDir := filepath.Join(g.googleCloudDir, "internal", "generated", "snippets")
if err := gensnippets.Generate(g.googleCloudDir, snippetDir); err != nil {
return fmt.Errorf("error generating snippets: %v", err)
}
if err := replaceAllForSnippets(g.googleCloudDir, snippetDir); err != nil {
return err
}
if err := goModTidy(snippetDir); err != nil {
return err
}

Expand All @@ -86,25 +100,85 @@ func (g *GapicGenerator) Regen(ctx context.Context) error {
return err
}

if err := g.dropModReplaceGenproto(); err != nil {
if err := forEachMod(g.googleCloudDir, g.dropModReplaceGenproto); err != nil {
return err
}

return nil
}

// forEachMod runs the given function with the directory of
// every non-internal module.
func forEachMod(rootDir string, fn func(dir string) error) error {
return filepath.WalkDir(rootDir, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if strings.Contains(path, "internal") {
return filepath.SkipDir
}
if d.Name() == "go.mod" {
if err := fn(filepath.Dir(path)); err != nil {
return err
}
}
return nil
})
}

func goModTidy(dir string) error {
log.Printf("[%s] running go mod tidy", dir)
c := command("go", "mod", "tidy")
c.Dir = dir
c.Env = []string{
fmt.Sprintf("PATH=%s", os.Getenv("PATH")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands.
fmt.Sprintf("HOME=%s", os.Getenv("HOME")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands.
}
return c.Run()
}

func replaceAllForSnippets(googleCloudDir, snippetDir string) error {
return forEachMod(googleCloudDir, func(dir string) error {
if dir == snippetDir {
return nil
}

// Get the module name in this dir.
modC := execabs.Command("go", "list", "-m")
modC.Dir = dir
mod, err := modC.Output()
if err != nil {
return err
}

// Replace it. Use a relative path to avoid issues on different systems.
rel, err := filepath.Rel(snippetDir, dir)
if err != nil {
return err
}
c := command("bash", "-c", `go mod edit -replace "$MODULE=$MODULE_PATH"`)
c.Dir = snippetDir
c.Env = []string{
fmt.Sprintf("PATH=%s", os.Getenv("PATH")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands.
fmt.Sprintf("HOME=%s", os.Getenv("HOME")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands.
fmt.Sprintf("MODULE=%s", mod),
fmt.Sprintf("MODULE_PATH=%s", rel),
}
return c.Run()
})
}

// addModReplaceGenproto adds a genproto replace statement that points genproto
// to the local copy. This is necessary since the remote genproto may not have
// changes that are necessary for the in-flight regen.
func (g *GapicGenerator) addModReplaceGenproto() error {
log.Println("adding temporary genproto replace statement")
func (g *GapicGenerator) addModReplaceGenproto(dir string) error {
log.Printf("[%s] adding temporary genproto replace statement", dir)
c := command("bash", "-c", `
set -ex
GENPROTO_VERSION=$(cat go.mod | cat go.mod | grep genproto | awk '{print $2}')
go mod edit -replace "google.golang.org/genproto@$GENPROTO_VERSION=$GENPROTO_DIR"
go mod edit -replace "google.golang.org/genproto=$GENPROTO_DIR"
`)
c.Dir = g.googleCloudDir
c.Dir = dir
c.Env = []string{
"GENPROTO_DIR=" + g.genprotoDir,
fmt.Sprintf("PATH=%s", os.Getenv("PATH")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands.
Expand All @@ -115,15 +189,14 @@ go mod edit -replace "google.golang.org/genproto@$GENPROTO_VERSION=$GENPROTO_DIR

// dropModReplaceGenproto drops the genproto replace statement. It is intended
// to be run after addModReplaceGenproto.
func (g *GapicGenerator) dropModReplaceGenproto() error {
log.Println("removing genproto replace statement")
func (g *GapicGenerator) dropModReplaceGenproto(dir string) error {
log.Printf("[%s] removing genproto replace statement", dir)
c := command("bash", "-c", `
set -ex
GENPROTO_VERSION=$(cat go.mod | cat go.mod | grep genproto | grep -v replace | awk '{print $2}')
go mod edit -dropreplace "google.golang.org/genproto@$GENPROTO_VERSION"
go mod edit -dropreplace "google.golang.org/genproto"
`)
c.Dir = g.googleCloudDir
c.Dir = dir
c.Env = []string{
fmt.Sprintf("PATH=%s", os.Getenv("PATH")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands.
fmt.Sprintf("HOME=%s", os.Getenv("HOME")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands.
Expand Down
13 changes: 7 additions & 6 deletions internal/gapicgen/go.mod
@@ -1,22 +1,23 @@
module cloud.google.com/go/internal/gapicgen

go 1.13
go 1.16

require (
cloud.google.com/go/internal/gensnippets v0.0.0-00010101000000-000000000000
github.com/golang/protobuf v1.5.2 // indirect
github.com/google/go-github/v34 v34.0.0
github.com/kr/text v0.2.0 // indirect
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
github.com/shurcooL/githubv4 v0.0.0-20201206200315-234843c633fa
github.com/shurcooL/graphql v0.0.0-20181231061246-d48a9a75455f // indirect
github.com/stretchr/testify v1.6.1 // indirect
golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4 // indirect
golang.org/x/oauth2 v0.0.0-20210413134643-5e61552d6c78
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57 // indirect
golang.org/x/text v0.3.5 // indirect
google.golang.org/appengine v1.6.7 // indirect
golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
gopkg.in/src-d/go-git.v4 v4.13.1
gopkg.in/yaml.v2 v2.4.0
)

replace cloud.google.com/go/internal/gensnippets => ../gensnippets

replace cloud.google.com/go/internal/godocfx => ../godocfx

0 comments on commit b70e0fc

Please sign in to comment.