From ef2b90ea6d47e27744c98a1a9ae0c487c5051808 Mon Sep 17 00:00:00 2001 From: Tyler Bui-Palsulich <26876514+tbpg@users.noreply.github.com> Date: Tue, 20 Apr 2021 11:33:40 -0400 Subject: [PATCH] feat(internal): generate region tags for snippets (#3962) I found a couple "manual" clients that needed repo-metadata entries, so I added them and included the updated JSON. There are many packages where it doesn't make sense to generate snippets with any sort of correct region tag. So, I just skipped them. We can come back if needed. The region tags end up slightly repetitive (e.g. accessapproval_generated_accessapproval_apiv1_Client_UpdateAccessApprovalSettings), but I think they're more predictable as-is because the package path may not always line up with the API shortname. More in depth configuration of the region tag would probably make things considerably more complex (need to associate every example with the underlying RPC?) --- internal/.repo-metadata-full.json | 16 +++++ internal/gapicgen/generator/gapics.go | 51 +++++++++++++- internal/gapicgen/gensnippets/gensnippets.go | 66 +++++++++++++++---- .../Client/ApproveApprovalRequest/main.go | 4 ++ .../DeleteAccessApprovalSettings/main.go | 4 ++ .../Client/DismissApprovalRequest/main.go | 4 ++ .../Client/GetAccessApprovalSettings/main.go | 4 ++ .../apiv1/Client/GetApprovalRequest/main.go | 4 ++ .../apiv1/Client/ListApprovalRequests/main.go | 4 ++ .../apiv1/Client/NewClient/main.go | 4 ++ .../UpdateAccessApprovalSettings/main.go | 4 ++ 11 files changed, 151 insertions(+), 14 deletions(-) diff --git a/internal/.repo-metadata-full.json b/internal/.repo-metadata-full.json index ecb5f8efda6..af753edf4ee 100644 --- a/internal/.repo-metadata-full.json +++ b/internal/.repo-metadata-full.json @@ -263,6 +263,14 @@ "docs_url": "https://pkg.go.dev/cloud.google.com/go/cloudtasks/apiv2beta3", "release_level": "beta" }, + "cloud.google.com/go/compute/metadata": { + "distribution_name": "cloud.google.com/go/compute/metadata", + "description": "Service Metadata API", + "language": "Go", + "client_library_type": "manual", + "docs_url": "https://pkg.go.dev/cloud.google.com/go/compute/metadata", + "release_level": "ga" + }, "cloud.google.com/go/container/apiv1": { "distribution_name": "cloud.google.com/go/container/apiv1", "description": "Kubernetes Engine API", @@ -455,6 +463,14 @@ "docs_url": "https://pkg.go.dev/cloud.google.com/go/functions/apiv1", "release_level": "ga" }, + "cloud.google.com/go/functions/metadata": { + "distribution_name": "cloud.google.com/go/functions/metadata", + "description": "Cloud Functions", + "language": "Go", + "client_library_type": "manual", + "docs_url": "https://pkg.go.dev/cloud.google.com/go/functions/metadata", + "release_level": "alpha" + }, "cloud.google.com/go/gaming/apiv1": { "distribution_name": "cloud.google.com/go/gaming/apiv1", "description": "Game Services API", diff --git a/internal/gapicgen/generator/gapics.go b/internal/gapicgen/generator/gapics.go index edf680f061f..1b1cdbe379d 100644 --- a/internal/gapicgen/generator/gapics.go +++ b/internal/gapicgen/generator/gapics.go @@ -82,7 +82,11 @@ func (g *GapicGenerator) Regen(ctx context.Context) error { } snippetDir := filepath.Join(g.googleCloudDir, "internal", "generated", "snippets") - if err := gensnippets.Generate(g.googleCloudDir, snippetDir); err != nil { + apiShortnames, err := g.parseAPIShortnames(microgenGapicConfigs, manualEntries) + if err != nil { + return err + } + if err := gensnippets.Generate(g.googleCloudDir, snippetDir, apiShortnames); err != nil { return fmt.Errorf("error generating snippets: %v", err) } if err := replaceAllForSnippets(g.googleCloudDir, snippetDir); err != nil { @@ -329,6 +333,22 @@ var manualEntries = []manifestEntry{ DocsURL: "https://pkg.go.dev/cloud.google.com/go/profiler", ReleaseLevel: "ga", }, + { + DistributionName: "cloud.google.com/go/compute/metadata", + Description: "Service Metadata API", + Language: "Go", + ClientLibraryType: "manual", + DocsURL: "https://pkg.go.dev/cloud.google.com/go/compute/metadata", + ReleaseLevel: "ga", + }, + { + DistributionName: "cloud.google.com/go/functions/metadata", + Description: "Cloud Functions", + Language: "Go", + ClientLibraryType: "manual", + DocsURL: "https://pkg.go.dev/cloud.google.com/go/functions/metadata", + ReleaseLevel: "alpha", + }, // Manuals with a GAPIC. { DistributionName: "cloud.google.com/go/errorreporting", @@ -425,3 +445,32 @@ func (g *GapicGenerator) copyMicrogenFiles() error { c.Dir = g.googleCloudDir return c.Run() } + +func (g *GapicGenerator) parseAPIShortnames(confs []*microgenConfig, manualEntries []manifestEntry) (map[string]string, error) { + shortnames := map[string]string{} + for _, conf := range confs { + yamlPath := filepath.Join(g.googleapisDir, conf.apiServiceConfigPath) + yamlFile, err := os.Open(yamlPath) + if err != nil { + return nil, err + } + config := struct { + Name string `yaml:"name"` + }{} + if err := yaml.NewDecoder(yamlFile).Decode(&config); err != nil { + return nil, fmt.Errorf("Decode: %v", err) + } + shortname := strings.TrimSuffix(config.Name, ".googleapis.com") + shortnames[conf.importPath] = shortname + } + + // Do our best for manuals. + for _, manual := range manualEntries { + p := strings.TrimPrefix(manual.DistributionName, "cloud.google.com/go/") + if strings.Contains(p, "/") { + p = p[0:strings.Index(p, "/")] + } + shortnames[manual.DistributionName] = p + } + return shortnames, nil +} diff --git a/internal/gapicgen/gensnippets/gensnippets.go b/internal/gapicgen/gensnippets/gensnippets.go index f97bdf261c5..44e1ce782d6 100644 --- a/internal/gapicgen/gensnippets/gensnippets.go +++ b/internal/gapicgen/gensnippets/gensnippets.go @@ -34,7 +34,7 @@ import ( ) // Generate reads all modules in rootDir and outputs their examples in outDir. -func Generate(rootDir, outDir string) error { +func Generate(rootDir, outDir string, apiShortnames map[string]string) error { if rootDir == "" { rootDir = "." } @@ -60,6 +60,7 @@ func Generate(rootDir, outDir string) error { log.Printf("Processing examples in %v directories: %q\n", len(dirs), dirs) trimPrefix := "cloud.google.com/go" + errs := []error{} for _, dir := range dirs { // Load does not look at nested modules. pis, err := pkgload.Load("./...", dir, nil) @@ -67,11 +68,14 @@ func Generate(rootDir, outDir string) error { return fmt.Errorf("failed to load packages: %v", err) } for _, pi := range pis { - if err := processExamples(pi.Doc, pi.Fset, trimPrefix, outDir); err != nil { - return fmt.Errorf("failed to process examples: %v", err) + if err := processExamples(pi.Doc, pi.Fset, trimPrefix, outDir, apiShortnames); err != nil { + errs = append(errs, fmt.Errorf("failed to process examples: %v", err)) } } } + if len(errs) > 0 { + log.Fatal(errs) + } if len(dirs) > 0 { cmd := execabs.Command("goimports", "-w", ".") @@ -84,11 +88,48 @@ func Generate(rootDir, outDir string) error { return nil } -func processExamples(pkg *doc.Package, fset *token.FileSet, trimPrefix, outDir string) error { +var skip = map[string]bool{ + "cloud.google.com/go": true, // No product for root package. + "cloud.google.com/go/civil": true, // General time/date package. + "cloud.google.com/go/cloudbuild/apiv1": true, // Has v2. + "cloud.google.com/go/cmd/go-cloud-debug-agent": true, // Command line tool. + "cloud.google.com/go/container": true, // Deprecated. + "cloud.google.com/go/containeranalysis/apiv1": true, // Accidental beta at wrong path? + "cloud.google.com/go/grafeas/apiv1": true, // With containeranalysis. + "cloud.google.com/go/httpreplay": true, // Helper. + "cloud.google.com/go/httpreplay/cmd/httpr": true, // Helper. + "cloud.google.com/go/longrunning": true, // Helper. + "cloud.google.com/go/monitoring/apiv3": true, // Has v2. + "cloud.google.com/go/translate": true, // Has newer version. +} + +func processExamples(pkg *doc.Package, fset *token.FileSet, trimPrefix, outDir string, apiShortnames map[string]string) error { + if skip[pkg.ImportPath] { + return nil + } trimmed := strings.TrimPrefix(pkg.ImportPath, trimPrefix) outDir = filepath.Join(outDir, trimmed) - regionTag := "generated" + strings.ReplaceAll(trimmed, "/", "_") + shortname, ok := apiShortnames[pkg.ImportPath] + if !ok { + // Do our best to find a shortname. For example, + // cloud.google.com/go/bigtable/bttest should lead to + // cloud.google.com/go/bigtable. + bestMatch := "" + for path := range apiShortnames { + if strings.HasPrefix(pkg.ImportPath, path) { + if len(path) > len(bestMatch) { + bestMatch = path + } + } + } + if bestMatch == "" { + return fmt.Errorf("could not find API shortname for %v", pkg.ImportPath) + } + log.Printf("The best match for %q is %q", pkg.ImportPath, bestMatch) + shortname = apiShortnames[bestMatch] + } + regionTag := shortname + "_generated" + strings.ReplaceAll(trimmed, "/", "_") // Note: variables and constants don't have examples. @@ -166,18 +207,17 @@ func writeExamples(outDir string, exs []*doc.Example, fset *token.FileSet, regio if _, err := f.WriteString(header()); err != nil { return err } - // TODO(tbpg): print the right region tag. - // tag := regionTag + "_" + ex.Name + tag := regionTag + "_" + ex.Name // Include an extra newline to keep separate from the package declaration. - // if _, err := fmt.Fprintf(f, "// [START %v]\n\n", tag); err != nil { - // return err - // } + if _, err := fmt.Fprintf(f, "// [START %v]\n\n", tag); err != nil { + return err + } if _, err := f.WriteString(s); err != nil { return err } - // if _, err := fmt.Fprintf(f, "// [END %v]\n", tag); err != nil { - // return err - // } + if _, err := fmt.Fprintf(f, "// [END %v]\n", tag); err != nil { + return err + } } return nil } diff --git a/internal/generated/snippets/accessapproval/apiv1/Client/ApproveApprovalRequest/main.go b/internal/generated/snippets/accessapproval/apiv1/Client/ApproveApprovalRequest/main.go index f45a765e510..41dbb63506e 100644 --- a/internal/generated/snippets/accessapproval/apiv1/Client/ApproveApprovalRequest/main.go +++ b/internal/generated/snippets/accessapproval/apiv1/Client/ApproveApprovalRequest/main.go @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +// [START accessapproval_generated_accessapproval_apiv1_Client_ApproveApprovalRequest] + package main import ( @@ -40,3 +42,5 @@ func main() { // TODO: Use resp. _ = resp } + +// [END accessapproval_generated_accessapproval_apiv1_Client_ApproveApprovalRequest] diff --git a/internal/generated/snippets/accessapproval/apiv1/Client/DeleteAccessApprovalSettings/main.go b/internal/generated/snippets/accessapproval/apiv1/Client/DeleteAccessApprovalSettings/main.go index 01b96962b60..78bd17cb7e8 100644 --- a/internal/generated/snippets/accessapproval/apiv1/Client/DeleteAccessApprovalSettings/main.go +++ b/internal/generated/snippets/accessapproval/apiv1/Client/DeleteAccessApprovalSettings/main.go @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +// [START accessapproval_generated_accessapproval_apiv1_Client_DeleteAccessApprovalSettings] + package main import ( @@ -36,3 +38,5 @@ func main() { // TODO: Handle error. } } + +// [END accessapproval_generated_accessapproval_apiv1_Client_DeleteAccessApprovalSettings] diff --git a/internal/generated/snippets/accessapproval/apiv1/Client/DismissApprovalRequest/main.go b/internal/generated/snippets/accessapproval/apiv1/Client/DismissApprovalRequest/main.go index 5fc38dbb270..756043d5dc6 100644 --- a/internal/generated/snippets/accessapproval/apiv1/Client/DismissApprovalRequest/main.go +++ b/internal/generated/snippets/accessapproval/apiv1/Client/DismissApprovalRequest/main.go @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +// [START accessapproval_generated_accessapproval_apiv1_Client_DismissApprovalRequest] + package main import ( @@ -40,3 +42,5 @@ func main() { // TODO: Use resp. _ = resp } + +// [END accessapproval_generated_accessapproval_apiv1_Client_DismissApprovalRequest] diff --git a/internal/generated/snippets/accessapproval/apiv1/Client/GetAccessApprovalSettings/main.go b/internal/generated/snippets/accessapproval/apiv1/Client/GetAccessApprovalSettings/main.go index f21164a3248..0a205feab97 100644 --- a/internal/generated/snippets/accessapproval/apiv1/Client/GetAccessApprovalSettings/main.go +++ b/internal/generated/snippets/accessapproval/apiv1/Client/GetAccessApprovalSettings/main.go @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +// [START accessapproval_generated_accessapproval_apiv1_Client_GetAccessApprovalSettings] + package main import ( @@ -40,3 +42,5 @@ func main() { // TODO: Use resp. _ = resp } + +// [END accessapproval_generated_accessapproval_apiv1_Client_GetAccessApprovalSettings] diff --git a/internal/generated/snippets/accessapproval/apiv1/Client/GetApprovalRequest/main.go b/internal/generated/snippets/accessapproval/apiv1/Client/GetApprovalRequest/main.go index 725f033d021..5a67e0fe999 100644 --- a/internal/generated/snippets/accessapproval/apiv1/Client/GetApprovalRequest/main.go +++ b/internal/generated/snippets/accessapproval/apiv1/Client/GetApprovalRequest/main.go @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +// [START accessapproval_generated_accessapproval_apiv1_Client_GetApprovalRequest] + package main import ( @@ -40,3 +42,5 @@ func main() { // TODO: Use resp. _ = resp } + +// [END accessapproval_generated_accessapproval_apiv1_Client_GetApprovalRequest] diff --git a/internal/generated/snippets/accessapproval/apiv1/Client/ListApprovalRequests/main.go b/internal/generated/snippets/accessapproval/apiv1/Client/ListApprovalRequests/main.go index a9c85b92135..e5c73079ca7 100644 --- a/internal/generated/snippets/accessapproval/apiv1/Client/ListApprovalRequests/main.go +++ b/internal/generated/snippets/accessapproval/apiv1/Client/ListApprovalRequests/main.go @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +// [START accessapproval_generated_accessapproval_apiv1_Client_ListApprovalRequests] + package main import ( @@ -48,3 +50,5 @@ func main() { _ = resp } } + +// [END accessapproval_generated_accessapproval_apiv1_Client_ListApprovalRequests] diff --git a/internal/generated/snippets/accessapproval/apiv1/Client/NewClient/main.go b/internal/generated/snippets/accessapproval/apiv1/Client/NewClient/main.go index eed78162a3c..3f6cf956730 100644 --- a/internal/generated/snippets/accessapproval/apiv1/Client/NewClient/main.go +++ b/internal/generated/snippets/accessapproval/apiv1/Client/NewClient/main.go @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +// [START accessapproval_generated_accessapproval_apiv1_NewClient] + package main import ( @@ -29,3 +31,5 @@ func main() { // TODO: Use client. _ = c } + +// [END accessapproval_generated_accessapproval_apiv1_NewClient] diff --git a/internal/generated/snippets/accessapproval/apiv1/Client/UpdateAccessApprovalSettings/main.go b/internal/generated/snippets/accessapproval/apiv1/Client/UpdateAccessApprovalSettings/main.go index a1d6c808eff..11421ed2508 100644 --- a/internal/generated/snippets/accessapproval/apiv1/Client/UpdateAccessApprovalSettings/main.go +++ b/internal/generated/snippets/accessapproval/apiv1/Client/UpdateAccessApprovalSettings/main.go @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +// [START accessapproval_generated_accessapproval_apiv1_Client_UpdateAccessApprovalSettings] + package main import ( @@ -40,3 +42,5 @@ func main() { // TODO: Use resp. _ = resp } + +// [END accessapproval_generated_accessapproval_apiv1_Client_UpdateAccessApprovalSettings]