From eaa742a248dc7d93c019863248f28e37f88aae84 Mon Sep 17 00:00:00 2001 From: Cody Oss <6331106+codyoss@users.noreply.github.com> Date: Fri, 4 Dec 2020 13:30:03 -0700 Subject: [PATCH] feat(internal/gapicgen): support generating only gapics with genlocal (#3383) This can speed up generation and removes the edge case where an error gets returned if there are no new proto stubs to generate since the last regen. --- internal/gapicgen/cmd/genbot/generate.go | 8 ++++++- internal/gapicgen/cmd/genlocal/main.go | 11 +++++++++- internal/gapicgen/generator/gapics.go | 26 ++++++++++++---------- internal/gapicgen/generator/generator.go | 28 +++++++++++++++++------- 4 files changed, 51 insertions(+), 22 deletions(-) diff --git a/internal/gapicgen/cmd/genbot/generate.go b/internal/gapicgen/cmd/genbot/generate.go index aa0725003e7..9604bc6a3a8 100644 --- a/internal/gapicgen/cmd/genbot/generate.go +++ b/internal/gapicgen/cmd/genbot/generate.go @@ -67,7 +67,13 @@ func generate(ctx context.Context, githubClient *GithubClient) error { } // Regen. - changes, err := generator.Generate(ctx, googleapisDir, genprotoDir, gocloudDir, protoDir, "") + conf := &generator.Config{ + GoogleapisDir: googleapisDir, + GenprotoDir: genprotoDir, + GapicDir: gocloudDir, + ProtoDir: protoDir, + } + changes, err := generator.Generate(ctx, conf) if err != nil { return err } diff --git a/internal/gapicgen/cmd/genlocal/main.go b/internal/gapicgen/cmd/genlocal/main.go index 98cfd3fa557..e80fc303eab 100644 --- a/internal/gapicgen/cmd/genlocal/main.go +++ b/internal/gapicgen/cmd/genlocal/main.go @@ -55,6 +55,7 @@ func main() { genprotoDir := flag.String("genproto-dir", filepath.Join(tmpDir, "genproto"), "Directory where sources of googleapis/go-genproto resides. If unset the sources will be cloned to a temporary directory that is not cleaned up.") protoDir := flag.String("proto-dir", filepath.Join(tmpDir, "proto"), "Directory where sources of google/protobuf resides. If unset the sources will be cloned to a temporary directory that is not cleaned up.") gapicToGenerate := flag.String("gapic", "", `Specifies which gapic to generate. The value should be in the form of an import path (Ex: cloud.google.com/go/pubsub/apiv1). The default "" generates all gapics.`) + onlyGapics := flag.Bool("only-gapics", false, "Enabling stops regenerating genproto.") verbose := flag.Bool("verbose", false, "Enables verbose logging.") flag.Parse() @@ -71,7 +72,15 @@ func main() { } // Regen. - changes, err := generator.Generate(ctx, *googleapisDir, *genprotoDir, *gocloudDir, *protoDir, *gapicToGenerate) + conf := &generator.Config{ + GoogleapisDir: *googleapisDir, + GenprotoDir: *genprotoDir, + GapicDir: *gocloudDir, + ProtoDir: *protoDir, + GapicToGenerate: *gapicToGenerate, + OnlyGenerateGapic: *onlyGapics, + } + changes, err := generator.Generate(ctx, conf) if err != nil { log.Printf("Generator ran (and failed) in %s\n", tmpDir) log.Fatal(err) diff --git a/internal/gapicgen/generator/gapics.go b/internal/gapicgen/generator/gapics.go index 4d960b0bd70..f2b42ae55e5 100644 --- a/internal/gapicgen/generator/gapics.go +++ b/internal/gapicgen/generator/gapics.go @@ -28,31 +28,33 @@ import ( // GapicGenerator is used to regenerate gapic libraries. type GapicGenerator struct { - googleapisDir string - protoDir string - googleCloudDir string - genprotoDir string + googleapisDir string + protoDir string + googleCloudDir string + genprotoDir string + gapicToGenerate string } // NewGapicGenerator creates a GapicGenerator. -func NewGapicGenerator(googleapisDir, protoDir, googleCloudDir, genprotoDir string) *GapicGenerator { +func NewGapicGenerator(googleapisDir, protoDir, googleCloudDir, genprotoDir string, gapicToGenerate string) *GapicGenerator { return &GapicGenerator{ - googleapisDir: googleapisDir, - protoDir: protoDir, - googleCloudDir: googleCloudDir, - genprotoDir: genprotoDir, + googleapisDir: googleapisDir, + protoDir: protoDir, + googleCloudDir: googleCloudDir, + genprotoDir: genprotoDir, + gapicToGenerate: gapicToGenerate, } } // Regen generates gapics. -func (g *GapicGenerator) Regen(ctx context.Context, gapicToGenerate string) error { +func (g *GapicGenerator) Regen(ctx context.Context) error { log.Println("regenerating gapics") for _, c := range microgenGapicConfigs { // Skip generation if generating all of the gapics and the associated // config has a block on it. Or if generating a single gapic and it does // not match the specified import path. - if (c.stopGeneration && gapicToGenerate == "") || - (gapicToGenerate != "" && gapicToGenerate != c.importPath) { + if (c.stopGeneration && g.gapicToGenerate == "") || + (g.gapicToGenerate != "" && g.gapicToGenerate != c.importPath) { continue } if err := g.microgen(c); err != nil { diff --git a/internal/gapicgen/generator/generator.go b/internal/gapicgen/generator/generator.go index 407c5cf5302..0099327d491 100644 --- a/internal/gapicgen/generator/generator.go +++ b/internal/gapicgen/generator/generator.go @@ -26,23 +26,35 @@ import ( "strings" ) +// Config contains inputs needed to generate sources. +type Config struct { + GoogleapisDir string + GenprotoDir string + GapicDir string + ProtoDir string + GapicToGenerate string + OnlyGenerateGapic bool +} + // Generate generates genproto and gapics. -func Generate(ctx context.Context, googleapisDir, genprotoDir, gocloudDir, protoDir string, gapicToGenerate string) ([]*ChangeInfo, error) { - protoGenerator := NewGenprotoGenerator(genprotoDir, googleapisDir, protoDir) - gapicGenerator := NewGapicGenerator(googleapisDir, protoDir, gocloudDir, genprotoDir) - if err := protoGenerator.Regen(ctx); err != nil { - return nil, fmt.Errorf("error generating genproto (may need to check logs for more errors): %v", err) +func Generate(ctx context.Context, conf *Config) ([]*ChangeInfo, error) { + if !conf.OnlyGenerateGapic { + protoGenerator := NewGenprotoGenerator(conf.GenprotoDir, conf.GoogleapisDir, conf.ProtoDir) + if err := protoGenerator.Regen(ctx); err != nil { + return nil, fmt.Errorf("error generating genproto (may need to check logs for more errors): %v", err) + } } - if err := gapicGenerator.Regen(ctx, gapicToGenerate); err != nil { + gapicGenerator := NewGapicGenerator(conf.GoogleapisDir, conf.ProtoDir, conf.GapicDir, conf.GenprotoDir, conf.GapicToGenerate) + if err := gapicGenerator.Regen(ctx); err != nil { return nil, fmt.Errorf("error generating gapics (may need to check logs for more errors): %v", err) } - changes, err := gatherChanges(googleapisDir, genprotoDir) + changes, err := gatherChanges(conf.GoogleapisDir, conf.GenprotoDir) if err != nil { return nil, fmt.Errorf("error gathering commit info") } - if err := recordGoogleapisHash(googleapisDir, genprotoDir); err != nil { + if err := recordGoogleapisHash(conf.GoogleapisDir, conf.GenprotoDir); err != nil { return nil, err }