chore(gapicgen): remove googleapis-discovery (#5109)

internal/gapicgen/cmd/genbot/generate.go

@@ -43,7 +43,6 @@ func generate(ctx context.Context, githubClient *git.GithubClient, forceAll bool
 	log.Printf("working out %s\n", tmpDir)
 
 	googleapisDir := filepath.Join(tmpDir, "googleapis")
-	googleapisDiscoDir := filepath.Join(tmpDir, "googleapis-discovery")
 	gocloudDir := filepath.Join(tmpDir, "gocloud")
 	genprotoDir := filepath.Join(tmpDir, "genproto")
 	protoDir := filepath.Join(tmpDir, "proto")
@@ -54,9 +53,6 @@ func generate(ctx context.Context, githubClient *git.GithubClient, forceAll bool
 	grp.Go(func() error {
 		return git.DeepClone("https://github.com/googleapis/googleapis", googleapisDir)
 	})
-	grp.Go(func() error {
-		return git.DeepClone("https://github.com/googleapis/googleapis-discovery", googleapisDiscoDir)
-	})
 	grp.Go(func() error {
 		return git.DeepClone("https://github.com/googleapis/go-genproto", genprotoDir)
 	})
@@ -72,12 +68,11 @@ func generate(ctx context.Context, githubClient *git.GithubClient, forceAll bool
 
 	// Regen.
 	conf := &generator.Config{
-		GoogleapisDir:      googleapisDir,
-		GoogleapisDiscoDir: googleapisDiscoDir,
-		GenprotoDir:        genprotoDir,
-		GapicDir:           gocloudDir,
-		ProtoDir:           protoDir,
-		ForceAll:           forceAll,
+		GoogleapisDir: googleapisDir,
+		GenprotoDir:   genprotoDir,
+		GapicDir:      gocloudDir,
+		ProtoDir:      protoDir,
+		ForceAll:      forceAll,
 	}
 	changes, err := generator.Generate(ctx, conf)
 	if err != nil {

internal/gapicgen/cmd/genbot/local.go

@@ -30,16 +30,15 @@ import (
 )
 
 type localConfig struct {
-	googleapisDir      string
-	googleapisDiscoDir string
-	gocloudDir         string
-	genprotoDir        string
-	protoDir           string
-	gapicToGenerate    string
-	onlyGapics         bool
-	regenOnly          bool
-	forceAll           bool
-	genModule          bool
+	googleapisDir   string
+	gocloudDir      string
+	genprotoDir     string
+	protoDir        string
+	gapicToGenerate string
+	onlyGapics      bool
+	regenOnly       bool
+	forceAll        bool
+	genModule       bool
 }
 
 func genLocal(ctx context.Context, c localConfig) error {
@@ -50,15 +49,13 @@ func genLocal(ctx context.Context, c localConfig) error {
 	}
 	log.Printf("temp dir created at %s\n", tmpDir)
 	tmpGoogleapisDir := filepath.Join(tmpDir, "googleapis")
-	tmpGoogleapisDiscoDir := filepath.Join(tmpDir, "googleapis-discovery")
 	tmpGenprotoDir := filepath.Join(tmpDir, "genproto")
 	tmpGocloudDir := filepath.Join(tmpDir, "gocloud")
 	tmpProtoDir := filepath.Join(tmpDir, "proto")
 
 	// Clone repositories if needed.
 	grp, _ := errgroup.WithContext(ctx)
 	gitShallowClone(grp, "https://github.com/googleapis/googleapis.git", c.googleapisDir, tmpGoogleapisDir)
-	gitShallowClone(grp, "https://github.com/googleapis/googleapis-discovery.git", c.googleapisDiscoDir, tmpGoogleapisDiscoDir)
 	gitShallowClone(grp, "https://github.com/googleapis/go-genproto", c.genprotoDir, tmpGenprotoDir)
 	gitShallowClone(grp, "https://github.com/googleapis/google-cloud-go", c.gocloudDir, tmpGocloudDir)
 	gitShallowClone(grp, "https://github.com/protocolbuffers/protobuf", c.protoDir, tmpProtoDir)
@@ -68,17 +65,16 @@ func genLocal(ctx context.Context, c localConfig) error {
 
 	// Regen.
 	conf := &generator.Config{
-		GoogleapisDir:      deafultDir(tmpGoogleapisDir, c.googleapisDir),
-		GoogleapisDiscoDir: deafultDir(tmpGoogleapisDiscoDir, c.googleapisDiscoDir),
-		GenprotoDir:        deafultDir(tmpGenprotoDir, c.genprotoDir),
-		GapicDir:           deafultDir(tmpGocloudDir, c.gocloudDir),
-		ProtoDir:           deafultDir(tmpProtoDir, c.protoDir),
-		GapicToGenerate:    c.gapicToGenerate,
-		OnlyGenerateGapic:  c.onlyGapics,
-		LocalMode:          true,
-		RegenOnly:          c.regenOnly,
-		ForceAll:           c.forceAll,
-		GenModule:          c.genModule,
+		GoogleapisDir:     deafultDir(tmpGoogleapisDir, c.googleapisDir),
+		GenprotoDir:       deafultDir(tmpGenprotoDir, c.genprotoDir),
+		GapicDir:          deafultDir(tmpGocloudDir, c.gocloudDir),
+		ProtoDir:          deafultDir(tmpProtoDir, c.protoDir),
+		GapicToGenerate:   c.gapicToGenerate,
+		OnlyGenerateGapic: c.onlyGapics,
+		LocalMode:         true,
+		RegenOnly:         c.regenOnly,
+		ForceAll:          c.forceAll,
+		GenModule:         c.genModule,
 	}
 	if _, err := generator.Generate(ctx, conf); err != nil {
 		log.Printf("Generator ran (and failed) in %s\n", tmpDir)

internal/gapicgen/cmd/genbot/main.go

@@ -46,7 +46,6 @@ func main() {
 
 	// flags for local mode
 	googleapisDir := flag.String("googleapis-dir", os.Getenv("GOOGLEAPIS_DIR"), "Directory where sources of googleapis/googleapis resides. If unset the sources will be cloned to a temporary directory that is not cleaned up.")
-	googleapisDiscoDir := flag.String("googleapis-disco-dir", os.Getenv("GOOGLEAPIS_DISCO_DIR"), "Directory where sources of googleapis/googleapis-discovery resides. If unset the sources will be cloned to a temporary directory that is not cleaned up.")
 	gocloudDir := flag.String("gocloud-dir", os.Getenv("GOCLOUD_DIR"), "Directory where sources of googleapis/google-cloud-go resides. If unset the sources will be cloned to a temporary directory that is not cleaned up.")
 	genprotoDir := flag.String("genproto-dir", os.Getenv("GENPROTO_DIR"), "Directory where sources of googleapis/go-genproto resides. If unset the sources will be cloned to a temporary directory that is not cleaned up.")
 	protoDir := flag.String("proto-dir", os.Getenv("PROTO_DIR"), "Directory where sources of google/protobuf resides. If unset the sources will be cloned to a temporary directory that is not cleaned up.")
@@ -59,16 +58,15 @@ func main() {
 
 	if *localMode {
 		if err := genLocal(ctx, localConfig{
-			googleapisDir:      *googleapisDir,
-			googleapisDiscoDir: *googleapisDiscoDir,
-			gocloudDir:         *gocloudDir,
-			genprotoDir:        *genprotoDir,
-			protoDir:           *protoDir,
-			gapicToGenerate:    *gapicToGenerate,
-			onlyGapics:         *onlyGapics,
-			regenOnly:          *regenOnly,
-			forceAll:           *forceAll,
-			genModule:          *genModule,
+			googleapisDir:   *googleapisDir,
+			gocloudDir:      *gocloudDir,
+			genprotoDir:     *genprotoDir,
+			protoDir:        *protoDir,
+			gapicToGenerate: *gapicToGenerate,
+			onlyGapics:      *onlyGapics,
+			regenOnly:       *regenOnly,
+			forceAll:        *forceAll,
+			genModule:       *genModule,
 		}); err != nil {
 			log.Fatal(err)
 		}

internal/gapicgen/generator/config.go

@@ -50,10 +50,6 @@ type microgenConfig struct {
 	// transports is a list of transports to generate a client for. Acceptable
 	// values are 'grpc' and 'rest'
 	transports []string
-
-	// googleapisDiscovery indicates if the protos reside in googleapis-discovery
-	// or not. Default is false, and will be looked up in googleapis.
-	googleapisDiscovery bool
 }
 
 var microgenGapicConfigs = []*microgenConfig{
@@ -65,8 +61,7 @@ var microgenGapicConfigs = []*microgenConfig{
 		apiServiceConfigPath: "compute_v1.yaml",
 		transports:           []string{"rest"},
 		// TODO: Change to "ga" when ready.
-		releaseLevel:        "beta",
-		googleapisDiscovery: true,
+		releaseLevel: "beta",
 	},
 	{
 		inputDirectoryPath:    "google/cloud/texttospeech/v1",

internal/gapicgen/generator/gapics.go

@@ -41,31 +41,29 @@ var (
 
 // GapicGenerator is used to regenerate gapic libraries.
 type GapicGenerator struct {
-	googleapisDir      string
-	googleapisDiscoDir string
-	protoDir           string
-	googleCloudDir     string
-	genprotoDir        string
-	gapicToGenerate    string
-	regenOnly          bool
-	onlyGenerateGapic  bool
-	genModule          bool
-	modifiedPkgs       []string
+	googleapisDir     string
+	protoDir          string
+	googleCloudDir    string
+	genprotoDir       string
+	gapicToGenerate   string
+	regenOnly         bool
+	onlyGenerateGapic bool
+	genModule         bool
+	modifiedPkgs      []string
 }
 
 // NewGapicGenerator creates a GapicGenerator.
 func NewGapicGenerator(c *Config, modifiedPkgs []string) *GapicGenerator {
 	return &GapicGenerator{
-		googleapisDir:      c.GoogleapisDir,
-		googleapisDiscoDir: c.GoogleapisDiscoDir,
-		protoDir:           c.ProtoDir,
-		googleCloudDir:     c.GapicDir,
-		genprotoDir:        c.GenprotoDir,
-		gapicToGenerate:    c.GapicToGenerate,
-		regenOnly:          c.RegenOnly,
-		onlyGenerateGapic:  c.OnlyGenerateGapic,
-		genModule:          c.GenModule,
-		modifiedPkgs:       modifiedPkgs,
+		googleapisDir:     c.GoogleapisDir,
+		protoDir:          c.ProtoDir,
+		googleCloudDir:    c.GapicDir,
+		genprotoDir:       c.GenprotoDir,
+		gapicToGenerate:   c.GapicToGenerate,
+		regenOnly:         c.RegenOnly,
+		onlyGenerateGapic: c.OnlyGenerateGapic,
+		genModule:         c.GenModule,
+		modifiedPkgs:      modifiedPkgs,
 	}
 }
 
@@ -315,13 +313,9 @@ find . -name '*.backup' -delete
 // microgen runs the microgenerator on a single microgen config.
 func (g *GapicGenerator) microgen(conf *microgenConfig) error {
 	log.Println("microgen generating", conf.pkg)
-	dir := g.googleapisDir
-	if conf.googleapisDiscovery {
-		dir = g.googleapisDiscoDir
-	}
 
 	var protoFiles []string
-	if err := filepath.Walk(dir+"/"+conf.inputDirectoryPath, func(path string, info os.FileInfo, err error) error {
+	if err := filepath.Walk(g.googleapisDir+"/"+conf.inputDirectoryPath, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
@@ -337,7 +331,6 @@ func (g *GapicGenerator) microgen(conf *microgenConfig) error {
 	}
 
 	args := []string{"-I", g.googleapisDir,
-		"-I", g.googleapisDiscoDir,
 		"--experimental_allow_proto3_optional",
 		"-I", g.protoDir,
 		"--go_gapic_out", g.googleCloudDir,
@@ -356,12 +349,13 @@ func (g *GapicGenerator) microgen(conf *microgenConfig) error {
 	if len(conf.transports) > 0 {
 		args = append(args, "--go_gapic_opt", fmt.Sprintf("transport=%s", strings.Join(conf.transports, "+")))
 	}
-	if conf.googleapisDiscovery {
+	// This is a bummer way of toggling diregapic generation, but it compute is the only one for the near term.
+	if conf.pkg == "compute" {
 		args = append(args, "--go_gapic_opt", "diregapic")
 	}
 	args = append(args, protoFiles...)
 	c := execv.Command("protoc", args...)
-	c.Dir = dir
+	c.Dir = g.googleapisDir
 	return c.Run()
 }
 
@@ -540,11 +534,7 @@ func (g *GapicGenerator) manifest(confs []*microgenConfig) (map[string]manifestE
 		entries[manual.DistributionName] = manual
 	}
 	for _, conf := range confs {
-		dir := g.googleapisDir
-		if conf.googleapisDiscovery {
-			dir = g.googleapisDiscoDir
-		}
-		yamlPath := filepath.Join(dir, conf.inputDirectoryPath, conf.apiServiceConfigPath)
+		yamlPath := filepath.Join(g.googleapisDir, conf.inputDirectoryPath, conf.apiServiceConfigPath)
 		yamlFile, err := os.Open(yamlPath)
 		if err != nil {
 			return nil, err
@@ -592,11 +582,7 @@ func (g *GapicGenerator) copyMicrogenFiles() error {
 func (g *GapicGenerator) parseAPIShortnames(confs []*microgenConfig, manualEntries []manifestEntry) (map[string]string, error) {
 	shortnames := map[string]string{}
 	for _, conf := range confs {
-		dir := g.googleapisDir
-		if conf.googleapisDiscovery {
-			dir = g.googleapisDiscoDir
-		}
-		yamlPath := filepath.Join(dir, conf.inputDirectoryPath, conf.apiServiceConfigPath)
+		yamlPath := filepath.Join(g.googleapisDir, conf.inputDirectoryPath, conf.apiServiceConfigPath)
 		yamlFile, err := os.Open(yamlPath)
 		if err != nil {
 			return nil, err

internal/gapicgen/generator/generator.go

@@ -30,17 +30,16 @@ import (
 
 // Config contains inputs needed to generate sources.
 type Config struct {
-	GoogleapisDir      string
-	GoogleapisDiscoDir string
-	GenprotoDir        string
-	GapicDir           string
-	ProtoDir           string
-	GapicToGenerate    string
-	OnlyGenerateGapic  bool
-	LocalMode          bool
-	RegenOnly          bool
-	ForceAll           bool
-	GenModule          bool
+	GoogleapisDir     string
+	GenprotoDir       string
+	GapicDir          string
+	ProtoDir          string
+	GapicToGenerate   string
+	OnlyGenerateGapic bool
+	LocalMode         bool
+	RegenOnly         bool
+	ForceAll          bool
+	GenModule         bool
 }
 
 // Generate generates genproto and gapics.

internal/gapicgen/generator/genproto.go

@@ -53,23 +53,26 @@ var denylist = map[string]bool{
 	"google.golang.org/genproto/googleapis/cloud/ondemandscanning/v1":      true,
 }
 
+// noGRPC is the set of APIs that do not need gRPC stubs.
+var noGRPC = map[string]bool{
+	"google.golang.org/genproto/googleapis/cloud/compute/v1": true,
+}
+
 // GenprotoGenerator is used to generate code for googleapis/go-genproto.
 type GenprotoGenerator struct {
-	genprotoDir        string
-	googleapisDir      string
-	googleapisDiscoDir string
-	protoSrcDir        string
-	forceAll           bool
+	genprotoDir   string
+	googleapisDir string
+	protoSrcDir   string
+	forceAll      bool
 }
 
 // NewGenprotoGenerator creates a new GenprotoGenerator.
 func NewGenprotoGenerator(c *Config) *GenprotoGenerator {
 	return &GenprotoGenerator{
-		genprotoDir:        c.GenprotoDir,
-		googleapisDir:      c.GoogleapisDir,
-		googleapisDiscoDir: c.GoogleapisDiscoDir,
-		protoSrcDir:        filepath.Join(c.ProtoDir, "/src"),
-		forceAll:           c.ForceAll,
+		genprotoDir:   c.GenprotoDir,
+		googleapisDir: c.GoogleapisDir,
+		protoSrcDir:   filepath.Join(c.ProtoDir, "/src"),
+		forceAll:      c.ForceAll,
 	}
 }
 
@@ -134,19 +137,14 @@ func (g *GenprotoGenerator) Regen(ctx context.Context) error {
 		if !strings.HasPrefix(pkg, "google.golang.org/genproto") || denylist[pkg] || hasPrefix(pkg, skipPrefixes) {
 			continue
 		}
+		grpc := !noGRPC[pkg]
 		pk := pkg
 		fn := fileNames
 		grp.Go(func() error {
 			log.Println("running protoc on", pk)
-			return g.protoc(fn, true /* grpc */)
+			return g.protoc(fn, grpc)
 		})
 	}
-	// TODO(noahdietz): This needs to be generalized to support any proto in googleapis-discovery.
-	// It's hard because the regen.txt contains the committish from googleapis last used to regen.
-	grp.Go(func() error {
-		log.Println("running protoc on compute")
-		return g.protoc([]string{"google/cloud/compute/v1/compute.proto"}, false /* grpc */)
-	})
 	if err := grp.Wait(); err != nil {
 		return err
 	}
@@ -195,7 +193,7 @@ func (g *GenprotoGenerator) protoc(fileNames []string, grpc bool) error {
 	if grpc {
 		stubs = fmt.Sprintf("--go_out=plugins=grpc:%s/generated", g.genprotoDir)
 	}
-	args := []string{"--experimental_allow_proto3_optional", stubs, "-I", g.googleapisDiscoDir, "-I", g.googleapisDir, "-I", g.protoSrcDir}
+	args := []string{"--experimental_allow_proto3_optional", stubs, "-I", g.googleapisDir, "-I", g.protoSrcDir}
 	args = append(args, fileNames...)
 	c := execv.Command("protoc", args...)
 	c.Dir = g.genprotoDir
@@ -217,6 +215,9 @@ func (g *GenprotoGenerator) getUpdatedPackages(googleapisHash string) (map[strin
 		if !strings.HasSuffix(v, ".proto") {
 			continue
 		}
+		if !strings.HasSuffix(v, "compute_small.proto") {
+			continue
+		}
 		path := filepath.Join(g.googleapisDir, v)
 		pkg, err := goPkg(path)
 		if err != nil {