chore(internal): gen googleapis-discovery compute genproto (#4306)

internal/gapicgen/cmd/genbot/generate.go

@@ -42,6 +42,7 @@ func generate(ctx context.Context, githubClient *git.GithubClient, forceAll bool
 	log.Printf("working out %s\n", tmpDir)
 
 	googleapisDir := filepath.Join(tmpDir, "googleapis")
+	googleapisDiscoDir := filepath.Join(tmpDir, "googleapis-discovery")
 	gocloudDir := filepath.Join(tmpDir, "gocloud")
 	genprotoDir := filepath.Join(tmpDir, "genproto")
 	protoDir := filepath.Join(tmpDir, "proto")
@@ -52,6 +53,9 @@ func generate(ctx context.Context, githubClient *git.GithubClient, forceAll bool
 	grp.Go(func() error {
 		return git.DeepClone("https://github.com/googleapis/googleapis", googleapisDir)
 	})
+	grp.Go(func() error {
+		return git.DeepClone("https://github.com/googleapis/googleapis-discovery", googleapisDiscoDir)
+	})
 	grp.Go(func() error {
 		return git.DeepClone("https://github.com/googleapis/go-genproto", genprotoDir)
 	})
@@ -67,11 +71,12 @@ func generate(ctx context.Context, githubClient *git.GithubClient, forceAll bool
 
 	// Regen.
 	conf := &generator.Config{
-		GoogleapisDir: googleapisDir,
-		GenprotoDir:   genprotoDir,
-		GapicDir:      gocloudDir,
-		ProtoDir:      protoDir,
-		ForceAll:      forceAll,
+		GoogleapisDir:      googleapisDir,
+		GoogleapisDiscoDir: googleapisDiscoDir,
+		GenprotoDir:        genprotoDir,
+		GapicDir:           gocloudDir,
+		ProtoDir:           protoDir,
+		ForceAll:           forceAll,
 	}
 	changes, err := generator.Generate(ctx, conf)
 	if err != nil {

internal/gapicgen/cmd/genbot/local.go

@@ -29,13 +29,14 @@ import (
 )
 
 type localConfig struct {
-	googleapisDir   string
-	gocloudDir      string
-	genprotoDir     string
-	protoDir        string
-	gapicToGenerate string
-	onlyGapics      bool
-	regenOnly       bool
+	googleapisDir      string
+	googleapisDiscoDir string
+	gocloudDir         string
+	genprotoDir        string
+	protoDir           string
+	gapicToGenerate    string
+	onlyGapics         bool
+	regenOnly          bool
 }
 
 func genLocal(ctx context.Context, c localConfig) error {
@@ -46,13 +47,15 @@ func genLocal(ctx context.Context, c localConfig) error {
 	}
 	log.Printf("temp dir created at %s\n", tmpDir)
 	tmpGoogleapisDir := filepath.Join(tmpDir, "googleapis")
+	tmpGoogleapisDiscoDir := filepath.Join(tmpDir, "googleapis-discovery")
 	tmpGenprotoDir := filepath.Join(tmpDir, "genproto")
 	tmpGocloudDir := filepath.Join(tmpDir, "gocloud")
 	tmpProtoDir := filepath.Join(tmpDir, "proto")
 
 	// Clone repositories if needed.
 	grp, _ := errgroup.WithContext(ctx)
 	gitShallowClone(grp, "https://github.com/googleapis/googleapis.git", c.googleapisDir, tmpGoogleapisDir)
+	gitShallowClone(grp, "https://github.com/googleapis/googleapis-discovery.git", c.googleapisDiscoDir, tmpGoogleapisDiscoDir)
 	gitShallowClone(grp, "https://github.com/googleapis/go-genproto", c.genprotoDir, tmpGenprotoDir)
 	gitShallowClone(grp, "https://github.com/googleapis/google-cloud-go", c.gocloudDir, tmpGocloudDir)
 	gitShallowClone(grp, "https://github.com/protocolbuffers/protobuf", c.protoDir, tmpProtoDir)
@@ -62,14 +65,15 @@ func genLocal(ctx context.Context, c localConfig) error {
 
 	// Regen.
 	conf := &generator.Config{
-		GoogleapisDir:     deafultDir(tmpGoogleapisDir, c.googleapisDir),
-		GenprotoDir:       deafultDir(tmpGenprotoDir, c.genprotoDir),
-		GapicDir:          deafultDir(tmpGocloudDir, c.gocloudDir),
-		ProtoDir:          deafultDir(tmpProtoDir, c.protoDir),
-		GapicToGenerate:   c.gapicToGenerate,
-		OnlyGenerateGapic: c.onlyGapics,
-		LocalMode:         true,
-		RegenOnly:         c.regenOnly,
+		GoogleapisDir:      deafultDir(tmpGoogleapisDir, c.googleapisDir),
+		GoogleapisDiscoDir: deafultDir(tmpGoogleapisDiscoDir, c.googleapisDiscoDir),
+		GenprotoDir:        deafultDir(tmpGenprotoDir, c.genprotoDir),
+		GapicDir:           deafultDir(tmpGocloudDir, c.gocloudDir),
+		ProtoDir:           deafultDir(tmpProtoDir, c.protoDir),
+		GapicToGenerate:    c.gapicToGenerate,
+		OnlyGenerateGapic:  c.onlyGapics,
+		LocalMode:          true,
+		RegenOnly:          c.regenOnly,
 	}
 	if _, err := generator.Generate(ctx, conf); err != nil {
 		log.Printf("Generator ran (and failed) in %s\n", tmpDir)

internal/gapicgen/cmd/genbot/main.go

@@ -45,6 +45,7 @@ func main() {
 
 	// flags for local mode
 	googleapisDir := flag.String("googleapis-dir", os.Getenv("GOOGLEAPIS_DIR"), "Directory where sources of googleapis/googleapis resides. If unset the sources will be cloned to a temporary directory that is not cleaned up.")
+	googleapisDiscoDir := flag.String("googleapis-disco-dir", os.Getenv("GOOGLEAPIS_DISCO_DIR"), "Directory where sources of googleapis/googleapis-discovery resides. If unset the sources will be cloned to a temporary directory that is not cleaned up.")
 	gocloudDir := flag.String("gocloud-dir", os.Getenv("GOCLOUD_DIR"), "Directory where sources of googleapis/google-cloud-go resides. If unset the sources will be cloned to a temporary directory that is not cleaned up.")
 	genprotoDir := flag.String("genproto-dir", os.Getenv("GENPROTO_DIR"), "Directory where sources of googleapis/go-genproto resides. If unset the sources will be cloned to a temporary directory that is not cleaned up.")
 	protoDir := flag.String("proto-dir", os.Getenv("PROTO_DIR"), "Directory where sources of google/protobuf resides. If unset the sources will be cloned to a temporary directory that is not cleaned up.")
@@ -56,13 +57,14 @@ func main() {
 
 	if *localMode {
 		if err := genLocal(ctx, localConfig{
-			googleapisDir:   *googleapisDir,
-			gocloudDir:      *gocloudDir,
-			genprotoDir:     *genprotoDir,
-			protoDir:        *protoDir,
-			gapicToGenerate: *gapicToGenerate,
-			onlyGapics:      *onlyGapics,
-			regenOnly:       *regenOnly,
+			googleapisDir:      *googleapisDir,
+			googleapisDiscoDir: *googleapisDiscoDir,
+			gocloudDir:         *gocloudDir,
+			genprotoDir:        *genprotoDir,
+			protoDir:           *protoDir,
+			gapicToGenerate:    *gapicToGenerate,
+			onlyGapics:         *onlyGapics,
+			regenOnly:          *regenOnly,
 		}); err != nil {
 			log.Fatal(err)
 		}

internal/gapicgen/generator/generator.go

@@ -29,15 +29,16 @@ import (
 
 // Config contains inputs needed to generate sources.
 type Config struct {
-	GoogleapisDir     string
-	GenprotoDir       string
-	GapicDir          string
-	ProtoDir          string
-	GapicToGenerate   string
-	OnlyGenerateGapic bool
-	LocalMode         bool
-	RegenOnly         bool
-	ForceAll          bool
+	GoogleapisDir      string
+	GoogleapisDiscoDir string
+	GenprotoDir        string
+	GapicDir           string
+	ProtoDir           string
+	GapicToGenerate    string
+	OnlyGenerateGapic  bool
+	LocalMode          bool
+	RegenOnly          bool
+	ForceAll           bool
 }
 
 // Generate generates genproto and gapics.

internal/gapicgen/generator/genproto.go

@@ -55,19 +55,21 @@ var denylist = map[string]bool{
 
 // GenprotoGenerator is used to generate code for googleapis/go-genproto.
 type GenprotoGenerator struct {
-	genprotoDir   string
-	googleapisDir string
-	protoSrcDir   string
-	forceAll      bool
+	genprotoDir        string
+	googleapisDir      string
+	googleapisDiscoDir string
+	protoSrcDir        string
+	forceAll           bool
 }
 
 // NewGenprotoGenerator creates a new GenprotoGenerator.
 func NewGenprotoGenerator(c *Config) *GenprotoGenerator {
 	return &GenprotoGenerator{
-		genprotoDir:   c.GenprotoDir,
-		googleapisDir: c.GoogleapisDir,
-		protoSrcDir:   filepath.Join(c.ProtoDir, "/src"),
-		forceAll:      c.ForceAll,
+		genprotoDir:        c.GenprotoDir,
+		googleapisDir:      c.GoogleapisDir,
+		googleapisDiscoDir: c.GoogleapisDiscoDir,
+		protoSrcDir:        filepath.Join(c.ProtoDir, "/src"),
+		forceAll:           c.ForceAll,
 	}
 }
 
@@ -102,7 +104,7 @@ func (g *GenprotoGenerator) Regen(ctx context.Context) error {
 	log.Println("regenerating genproto")
 
 	// Create space to put generated .pb.go's.
-	c := execv.Command("mkdir", "generated")
+	c := execv.Command("mkdir", "-p", "generated")
 	c.Dir = g.genprotoDir
 	if err := c.Run(); err != nil {
 		return err
@@ -114,6 +116,10 @@ func (g *GenprotoGenerator) Regen(ctx context.Context) error {
 		return err
 	}
 
+	// TODO(noahdietz): In local mode, since it clones a shallow copy with 1 commit,
+	// if the last regenerated hash is earlier than the top commit, the git diff-tree
+	// command fails. This is is a bit of a rough edge. Using my local clone of
+	// googleapis rectified the issue.
 	pkgFiles, err := g.getUpdatedPackages(string(lastHash))
 	if err != nil {
 		return err
@@ -132,9 +138,15 @@ func (g *GenprotoGenerator) Regen(ctx context.Context) error {
 		fn := fileNames
 		grp.Go(func() error {
 			log.Println("running protoc on", pk)
-			return g.protoc(fn)
+			return g.protoc(fn, true /* grpc */)
 		})
 	}
+	// TODO(noahdietz): This needs to be generalized to support any proto in googleapis-discovery.
+	// It's hard because the regen.txt contains the committish from googleapis last used to regen.
+	grp.Go(func() error {
+		log.Println("running protoc on compute")
+		return g.protoc([]string{"google/cloud/compute/v1/compute.proto"}, false /* grpc */)
+	})
 	if err := grp.Wait(); err != nil {
 		return err
 	}
@@ -178,8 +190,12 @@ func goPkg(fileName string) (string, error) {
 
 // protoc executes the "protoc" command on files named in fileNames, and outputs
 // to "<genprotoDir>/generated".
-func (g *GenprotoGenerator) protoc(fileNames []string) error {
-	args := []string{"--experimental_allow_proto3_optional", fmt.Sprintf("--go_out=plugins=grpc:%s/generated", g.genprotoDir), "-I", g.googleapisDir, "-I", g.protoSrcDir}
+func (g *GenprotoGenerator) protoc(fileNames []string, grpc bool) error {
+	stubs := fmt.Sprintf("--go_out=%s/generated", g.genprotoDir)
+	if grpc {
+		stubs = fmt.Sprintf("--go_out=plugins=grpc:%s/generated", g.genprotoDir)
+	}
+	args := []string{"--experimental_allow_proto3_optional", stubs, "-I", g.googleapisDiscoDir, "-I", g.googleapisDir, "-I", g.protoSrcDir}
 	args = append(args, fileNames...)
 	c := execv.Command("protoc", args...)
 	c.Dir = g.genprotoDir