diff --git a/internal/gapicgen/cmd/genbot/bot.go b/internal/gapicgen/cmd/genbot/bot.go index b75115b32b4..5ab3c3b3ff4 100644 --- a/internal/gapicgen/cmd/genbot/bot.go +++ b/internal/gapicgen/cmd/genbot/bot.go @@ -26,12 +26,20 @@ import ( "cloud.google.com/go/internal/gapicgen/git" ) -func genBot(ctx context.Context, githubAccessToken, githubUsername, githubName, githubEmail string) error { +type botConfig struct { + githubAccessToken string + githubUsername string + githubName string + githubEmail string + forceAll bool +} + +func genBot(ctx context.Context, c botConfig) error { for k, v := range map[string]string{ - "githubAccessToken": githubAccessToken, - "githubUsername": githubUsername, - "githubName": githubName, - "githubEmail": githubEmail, + "githubAccessToken": c.githubAccessToken, + "githubUsername": c.githubUsername, + "githubName": c.githubName, + "githubEmail": c.githubEmail, } { if v == "" { log.Printf("missing or empty value for required flag --%s\n", k) @@ -40,7 +48,7 @@ func genBot(ctx context.Context, githubAccessToken, githubUsername, githubName, } // Setup the client and git environment. - githubClient, err := git.NewGithubClient(ctx, githubUsername, githubName, githubEmail, githubAccessToken) + githubClient, err := git.NewGithubClient(ctx, c.githubUsername, c.githubName, c.githubEmail, c.githubAccessToken) if err != nil { return err } @@ -73,7 +81,7 @@ func genBot(ctx context.Context, githubAccessToken, githubUsername, githubName, return nil } - return generate(ctx, githubClient) + return generate(ctx, githubClient, c.forceAll) } // hasCreatedPRToday checks if the created time of a PR is from today. diff --git a/internal/gapicgen/cmd/genbot/generate.go b/internal/gapicgen/cmd/genbot/generate.go index 170090a34ba..ac6903866de 100644 --- a/internal/gapicgen/cmd/genbot/generate.go +++ b/internal/gapicgen/cmd/genbot/generate.go @@ -31,7 +31,7 @@ import ( // generate downloads sources and generates pull requests for go-genproto and // google-cloud-go if needed. -func generate(ctx context.Context, githubClient *git.GithubClient) error { +func generate(ctx context.Context, githubClient *git.GithubClient, forceAll bool) error { log.Println("creating temp dir") tmpDir, err := ioutil.TempDir("", "update-genproto") if err != nil { @@ -71,6 +71,7 @@ func generate(ctx context.Context, githubClient *git.GithubClient) error { GenprotoDir: genprotoDir, GapicDir: gocloudDir, ProtoDir: protoDir, + ForceAll: forceAll, } changes, err := generator.Generate(ctx, conf) if err != nil { diff --git a/internal/gapicgen/cmd/genbot/main.go b/internal/gapicgen/cmd/genbot/main.go index 7f90115eccf..ddaa4f302e1 100644 --- a/internal/gapicgen/cmd/genbot/main.go +++ b/internal/gapicgen/cmd/genbot/main.go @@ -41,6 +41,7 @@ func main() { githubName := flag.String("githubName", os.Getenv("GITHUB_NAME"), "The name of the author for git commits.") githubEmail := flag.String("githubEmail", os.Getenv("GITHUB_EMAIL"), "The email address of the author.") localMode := flag.Bool("local", strToBool(os.Getenv("GENBOT_LOCAL_MODE")), "Enables generating sources locally. This mode will not open any pull requests.") + forceAll := flag.Bool("forceAll", strToBool(os.Getenv("GENBOT_FORCE_ALL")), "Enables regenerating everything regardless of changes in googleapis.") // flags for local mode googleapisDir := flag.String("googleapis-dir", os.Getenv("GOOGLEAPIS_DIR"), "Directory where sources of googleapis/googleapis resides. If unset the sources will be cloned to a temporary directory that is not cleaned up.") @@ -67,7 +68,13 @@ func main() { } return } - if err := genBot(ctx, *githubAccessToken, *githubUsername, *githubName, *githubEmail); err != nil { + if err := genBot(ctx, botConfig{ + githubAccessToken: *githubAccessToken, + githubUsername: *githubUsername, + githubName: *githubName, + githubEmail: *githubEmail, + forceAll: *forceAll, + }); err != nil { log.Fatal(err) } } diff --git a/internal/gapicgen/generator/config.go b/internal/gapicgen/generator/config.go index 17afd1b2ebf..7c25b1acddd 100644 --- a/internal/gapicgen/generator/config.go +++ b/internal/gapicgen/generator/config.go @@ -1021,6 +1021,7 @@ var microgenGapicConfigs = []*microgenConfig{ gRPCServiceConfigPath: "google/cloud/recommendationengine/v1beta1/recommendationengine_grpc_service_config.json", apiServiceConfigPath: "google/cloud/recommendationengine/v1beta1/recommendationengine_v1beta1.yaml", releaseLevel: "beta", + stopGeneration: true, }, { inputDirectoryPath: "google/cloud/gkehub/v1beta1", diff --git a/internal/gapicgen/generator/generator.go b/internal/gapicgen/generator/generator.go index 1c17e0e43d9..fa644e43ca4 100644 --- a/internal/gapicgen/generator/generator.go +++ b/internal/gapicgen/generator/generator.go @@ -37,6 +37,7 @@ type Config struct { OnlyGenerateGapic bool LocalMode bool RegenOnly bool + ForceAll bool } // Generate generates genproto and gapics. diff --git a/internal/gapicgen/generator/genproto.go b/internal/gapicgen/generator/genproto.go index 0314bdf5a0e..33b1b7b6390 100644 --- a/internal/gapicgen/generator/genproto.go +++ b/internal/gapicgen/generator/genproto.go @@ -20,6 +20,7 @@ import ( "fmt" "io/ioutil" "log" + "os" "path/filepath" "regexp" "strconv" @@ -57,6 +58,7 @@ type GenprotoGenerator struct { genprotoDir string googleapisDir string protoSrcDir string + forceAll bool } // NewGenprotoGenerator creates a new GenprotoGenerator. @@ -65,6 +67,7 @@ func NewGenprotoGenerator(c *Config) *GenprotoGenerator { genprotoDir: c.GenprotoDir, googleapisDir: c.GoogleapisDir, protoSrcDir: filepath.Join(c.ProtoDir, "/src"), + forceAll: c.ForceAll, } } @@ -186,6 +189,9 @@ func (g *GenprotoGenerator) protoc(fileNames []string) error { // getUpdatedPackages parses all of the new commits to find what packages need // to be regenerated. func (g *GenprotoGenerator) getUpdatedPackages(googleapisHash string) (map[string][]string, error) { + if g.forceAll { + return g.getAllPackages() + } files, err := git.UpdateFilesSinceHash(g.googleapisDir, googleapisHash) if err != nil { return nil, err @@ -205,6 +211,41 @@ func (g *GenprotoGenerator) getUpdatedPackages(googleapisHash string) (map[strin return pkgFiles, nil } +func (g *GenprotoGenerator) getAllPackages() (map[string][]string, error) { + seenFiles := make(map[string]bool) + pkgFiles := make(map[string][]string) + for _, root := range []string{g.googleapisDir} { + walkFn := func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.Mode().IsRegular() || !strings.HasSuffix(path, ".proto") { + return nil + } + + switch rel, err := filepath.Rel(root, path); { + case err != nil: + return err + case seenFiles[rel]: + return nil + default: + seenFiles[rel] = true + } + + pkg, err := goPkg(path) + if err != nil { + return err + } + pkgFiles[pkg] = append(pkgFiles[pkg], path) + return nil + } + if err := filepath.Walk(root, walkFn); err != nil { + return nil, err + } + } + return pkgFiles, nil +} + // moveAndCleanupGeneratedSrc moves all generated src to their correct locations // in the repository, because protoc puts it in a folder called `generated/``. func (g *GenprotoGenerator) moveAndCleanupGeneratedSrc() error {