diff --git a/internal/carver/README.md b/internal/carver/README.md new file mode 100644 index 00000000000..10cdcd412cc --- /dev/null +++ b/internal/carver/README.md @@ -0,0 +1,12 @@ +# carver + +This is a tool used to carve out new modules in cloud.google.com/go. + +## Usage + +```bash +go run cmd/main.go \ + -parent=/path/to/google-cloud-go \ + -child=asset \ + -repo-metadata=/path/to/google-cloud-go/internal/.repo-metadata-full.json +``` diff --git a/internal/carver/cmd/_CHANGES.md.txt b/internal/carver/cmd/_CHANGES.md.txt new file mode 100644 index 00000000000..f1e8d07c3e3 --- /dev/null +++ b/internal/carver/cmd/_CHANGES.md.txt @@ -0,0 +1,6 @@ +# Changes + +## v0.1.0 + +This is the first tag to carve out {{.Package}} as its own module. See +[Add a module to a multi-module repository](https://github.com/golang/go/wiki/Modules#is-it-possible-to-add-a-module-to-a-multi-module-repository). diff --git a/internal/carver/cmd/_README.md.txt b/internal/carver/cmd/_README.md.txt new file mode 100644 index 00000000000..1d284c9f834 --- /dev/null +++ b/internal/carver/cmd/_README.md.txt @@ -0,0 +1,40 @@ +# {{.Name}} + +[![Go Reference](https://pkg.go.dev/badge/{{.ImportPath}}.svg)](https://pkg.go.dev/{{.ImportPath}}) + +Go Client Library for {{.Name}}. + +## Install + +```bash +go get {{.ImportPath}} +``` + +## Stability + +The stability of this module is indicated by SemVer. + +However, a `v1+` module may have breaking changes in two scenarios: + +* Packages with `alpha` or `beta` in the import path +* The GoDoc has an explicit stability disclaimer (for example, for an experimental feature). + +## Go Version Support + +See the [Go Versions Supported](https://github.com/googleapis/google-cloud-go#go-versions-supported) +section in the root directory's README. + +## Authorization + +See the [Authorization](https://github.com/googleapis/google-cloud-go#authorization) +section in the root directory's README. + +## Contributing + +Contributions are welcome. Please, see the [CONTRIBUTING](https://github.com/GoogleCloudPlatform/google-cloud-go/blob/master/CONTRIBUTING.md) +document for details. + +Please note that this project is released with a Contributor Code of Conduct. +By participating in this project you agree to abide by its terms. See +[Contributor Code of Conduct](https://github.com/GoogleCloudPlatform/google-cloud-go/blob/master/CONTRIBUTING.md#contributor-code-of-conduct) +for more information. diff --git a/internal/carver/cmd/_tidyhack_tmpl.txt b/internal/carver/cmd/_tidyhack_tmpl.txt new file mode 100644 index 00000000000..b35c2fd5c06 --- /dev/null +++ b/internal/carver/cmd/_tidyhack_tmpl.txt @@ -0,0 +1,22 @@ +// Copyright {{.Year}} Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file, and the {{.RootMod}} import, won't actually become part of +// the resultant binary. +// +build modhack + +package {{.Package}} + +// Necessary for safely adding multi-module repo. See: https://github.com/golang/go/wiki/Modules#is-it-possible-to-add-a-module-to-a-multi-module-repository +import _ "{{.RootMod}}" \ No newline at end of file diff --git a/internal/carver/cmd/main.go b/internal/carver/cmd/main.go new file mode 100644 index 00000000000..f2c90387394 --- /dev/null +++ b/internal/carver/cmd/main.go @@ -0,0 +1,452 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and + +package main + +import ( + "bytes" + _ "embed" + "encoding/json" + "flag" + "fmt" + "io" + "io/ioutil" + "log" + "os" + "os/exec" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" + "text/template" + "time" +) + +const ( + marjorWeight = 1e9 + minorWeight = 1e4 + patchWeight = 1 +) + +var ( + semverRegex = regexp.MustCompile(`.*v(?P\d*)\.(?P\d*)\.(?P\d*)(?P.*)`) + //go:embed _tidyhack_tmpl.txt + tidyHackTmpl string + //go:embed _CHANGES.md.txt + changesTmpl string + //go:embed _README.md.txt + readmeTmpl string +) + +type carver struct { + // flags + parentModPath string + parentGitTag string + parentGitTagPrefix string + childTagVersion string + childModPath string + repoMetadataPath string + name string + dryRun bool + + w io.WriteCloser +} + +func main() { + parent := flag.String("parent", "", "The path to the parent module. Required.") + child := flag.String("child", "", "The relative path to the child module from the parent module. Required.") + repoMetadataPath := flag.String("repo-metadata", "", "The full path to the repo metadata file. Required.") + name := flag.String("name", "", "The name used to identify the API in the README. Optional") + parentTagPrefix := flag.String("parent-tag-prefix", "", "The prefix for a git tag, should end in a '/'. Only required if parent is not the root module. Optional.") + parentTag := flag.String("parent-tag", "", "The newest tag from the parent module, this will override the lookup. If not specified the latest tag will be used. Optional.") + childTagVersion := flag.String("child-tag-version", "v0.1.0", "The tag version of the carved out child module. Should be in the form of vX.X.X with no prefix. Optional.") + dryRun := flag.Bool("dry-run", false, "If true no files or tags will be created. Optional.") + flag.Usage = func() { + fmt.Fprintf(flag.CommandLine.Output(), "Usage of %s:\n", os.Args[0]) + flag.PrintDefaults() + fmt.Fprintf(flag.CommandLine.Output(), "\nExample\n\tcarver -parent=/Users/me/google-cloud-go -child=asset repo-metadata=/Users/me/google-cloud-go/internal/.repo-metadata-full.json\n") + } + flag.Parse() + + c := &carver{ + parentModPath: *parent, + parentGitTag: *parentTag, + parentGitTagPrefix: *parentTagPrefix, + childTagVersion: *childTagVersion, + childModPath: filepath.Join(*parent, *child), + repoMetadataPath: *repoMetadataPath, + name: *name, + dryRun: *dryRun, + } + if err := c.Run(); err != nil { + log.Println(err) + flag.Usage() + os.Exit(1) + } + log.Println("Successfully carved out module. Changes are ready to be pushed.") +} + +func (c *carver) Run() error { + if c.parentModPath == "" || c.childModPath == "" || c.repoMetadataPath == "" { + return fmt.Errorf("all required flags were not provided") + } + rootMod, err := c.LookupParentModInfo() + if err != nil { + return fmt.Errorf("failed to lookup parent mod info: %v", err) + } + childPkgName, err := parsePkgName(c.childModPath) + if err != nil { + return err + } + if err := c.CreateChildCommonFiles(childPkgName, rootMod); err != nil { + return fmt.Errorf("failed to create readme: %v", err) + } + if err := c.CreateChildModule(childPkgName, rootMod); err != nil { + return fmt.Errorf("failed to create child module: %v", err) + } + if err := c.CreateGitTags(rootMod); err != nil { + return fmt.Errorf("failed to create child module: %v", err) + } + + return nil +} + +type modInfo struct { + filePath string + moduleName string + tag string +} + +func (c *carver) LookupParentModInfo() (*modInfo, error) { + log.Println("Looking up parent module import path") + cmd := exec.Command("go", "list", "-f", "{{.ImportPath}}") + cmd.Dir = c.parentModPath + b, err := cmd.Output() + if err != nil { + return nil, err + } + modName := string(bytes.TrimSpace(b)) + + if c.parentGitTag != "" { + return &modInfo{ + filePath: c.parentModPath, + moduleName: modName, + tag: c.parentGitTag, + }, nil + } + + log.Println("Looking up latest parent tag") + cmd = exec.Command("git", "tag") + cmd.Dir = c.parentModPath + b, err = cmd.Output() + if err != nil { + return nil, err + } + + var relevantTags []string + for _, tag := range strings.Split(string(bytes.TrimSpace(b)), "\n") { + if c.parentGitTagPrefix != "" && strings.HasPrefix(tag, c.parentGitTagPrefix) { + relevantTags = append(relevantTags, tag) + continue + } + if c.parentGitTagPrefix == "" && !strings.Contains(tag, "/") && strings.HasPrefix(tag, "v") { + relevantTags = append(relevantTags, tag) + } + } + sortTags(relevantTags) + tag := relevantTags[0] + log.Println("Found latest tag: ", tag) + + return &modInfo{ + filePath: c.parentModPath, + moduleName: modName, + tag: tag, + }, nil +} + +func (c *carver) CreateChildCommonFiles(pkgName string, rootMod *modInfo) error { + log.Printf("Reading metadata file from %q", c.repoMetadataPath) + metaFile, err := os.Open(c.repoMetadataPath) + if err != nil { + return fmt.Errorf("unable to open metadata file: %v", err) + } + meta, err := parseMetadata(metaFile) + if err != nil { + return err + } + + readmePath := filepath.Join(c.childModPath, "README.md") + log.Printf("Creating %q", readmePath) + readmeFile, err := c.newWriterCloser(readmePath) + if err != nil { + return err + } + defer readmeFile.Close() + t := template.Must(template.New("readme").Parse(readmeTmpl)) + importPath := rootMod.moduleName + strings.TrimPrefix(c.childModPath, rootMod.filePath) + name := c.name + if name == "" { + name = meta[importPath] + if name == "" { + return fmt.Errorf("unable to determine a name from API metadata, please set -name flag") + } + } + readmeData := struct { + Name string + ImportPath string + }{ + Name: name, + ImportPath: importPath, + } + if err := t.Execute(readmeFile, readmeData); err != nil { + return err + } + + changesPath := filepath.Join(c.childModPath, "CHANGES.md") + log.Printf("Creating %q", changesPath) + changesFile, err := c.newWriterCloser(changesPath) + if err != nil { + return err + } + defer changesFile.Close() + t2 := template.Must(template.New("changes").Parse(changesTmpl)) + changesData := struct { + Package string + }{ + Package: pkgName, + } + return t2.Execute(changesFile, changesData) +} + +func (c *carver) CreateChildModule(pkgName string, rootMod *modInfo) error { + fp := filepath.Join(c.childModPath, "go_mod_tidy_hack.go") + log.Printf("Creating %q", fp) + f, err := c.newWriterCloser(fp) + if err != nil { + return err + } + defer f.Close() + t := template.Must(template.New("tidyhack").Parse(tidyHackTmpl)) + data := struct { + Year int + RootMod string + Package string + }{ + Year: time.Now().Year(), + RootMod: rootMod.moduleName, + Package: pkgName, + } + if err := t.Execute(f, data); err != nil { + return err + } + + log.Printf("Creating child module in %q", c.childModPath) + if c.dryRun { + return nil + } + childModName := rootMod.moduleName + strings.TrimPrefix(c.childModPath, rootMod.filePath) + cmd := exec.Command("go", "mod", "init", childModName) + cmd.Dir = c.childModPath + if b, err := cmd.Output(); err != nil { + return fmt.Errorf("unable to init module: %s", b) + } + + futureTag, err := bumpSemverPatch(rootMod.tag) + if err != nil { + return err + } + cmd = exec.Command("go", "mod", "edit", "-require", fmt.Sprintf("%s@%s", rootMod.moduleName, futureTag)) + cmd.Dir = c.childModPath + if b, err := cmd.Output(); err != nil { + return fmt.Errorf("unable to require module: %s", b) + } + + cmd = exec.Command("go", "mod", "edit", "-replace", fmt.Sprintf("%s@%s=%s", rootMod.moduleName, futureTag, rootMod.filePath)) + cmd.Dir = c.childModPath + if b, err := cmd.Output(); err != nil { + return fmt.Errorf("unable to add replace module: %s", b) + } + + cmd = exec.Command("go", "mod", "tidy") + cmd.Dir = c.childModPath + if b, err := cmd.Output(); err != nil { + return fmt.Errorf("unable to tidy child module: %s", b) + } + + cmd = exec.Command("go", "mod", "edit", "-dropreplace", fmt.Sprintf("%s@%s", rootMod.moduleName, futureTag)) + cmd.Dir = c.childModPath + if b, err := cmd.Output(); err != nil { + return fmt.Errorf("unable to add replace module: %s", b) + } + + cmd = exec.Command("go", "mod", "tidy") + cmd.Dir = rootMod.filePath + if b, err := cmd.Output(); err != nil { + return fmt.Errorf("unable to tidy parent module: %s", b) + } + return nil +} + +func (c *carver) CreateGitTags(rootMod *modInfo) error { + futureTag, err := bumpSemverPatch(rootMod.tag) + if err != nil { + return err + } + childPrefix := strings.TrimPrefix(strings.TrimPrefix(c.childModPath, rootMod.filePath), "/") + log.Println("Commiting changes") + if !c.dryRun { + cmd := exec.Command("git", "add", "-A") + cmd.Dir = c.parentModPath + if b, err := cmd.Output(); err != nil { + return fmt.Errorf("unable to add changes: %s", b) + } + cmd = exec.Command("git", "commit", "-m", + fmt.Sprintf("chore(%s): carve out sub-module", childPrefix)) + cmd.Dir = c.parentModPath + if b, err := cmd.Output(); err != nil { + return fmt.Errorf("unable to commit changes: %s", b) + } + } + log.Printf("Tagging Root module: %s", futureTag) + if !c.dryRun { + cmd := exec.Command("git", "tag", futureTag) + cmd.Dir = c.parentModPath + if b, err := cmd.Output(); err != nil { + return fmt.Errorf("unable to tag root module: %s", b) + } + } + log.Printf("Tagging Child module: %s/%s", childPrefix, c.childTagVersion) + if !c.dryRun { + cmd := exec.Command("git", "tag", fmt.Sprintf("%s/%s", childPrefix, c.childTagVersion)) + cmd.Dir = c.parentModPath + if b, err := cmd.Output(); err != nil { + return fmt.Errorf("unable to tag child module: %s", b) + } + } + return nil +} + +// newWriterCloser is wrapper for creating a file. Used for testing and +// dry-runs. +func (c *carver) newWriterCloser(fp string) (io.WriteCloser, error) { + if c.dryRun { + return noopCloser{w: io.Discard}, nil + } + if c.w != nil { + return noopCloser{w: c.w}, nil + } + return os.Create(fp) +} + +// sortTags does a best effort sort based on semver. It was made a function for +// testing. Only the top result will ever be used. +func sortTags(tags []string) { + sort.Slice(tags, func(i, j int) bool { + imatch := semverRegex.FindStringSubmatch(tags[i]) + jmatch := semverRegex.FindStringSubmatch(tags[j]) + if len(imatch) < 5 { + return false + } + if len(jmatch) < 5 { + return true + } + + // Matches must be numbers due to regex they are parsed from. + iM, _ := strconv.Atoi(imatch[1]) + jM, _ := strconv.Atoi(jmatch[1]) + im, _ := strconv.Atoi(imatch[2]) + jm, _ := strconv.Atoi(jmatch[2]) + ip, _ := strconv.Atoi(imatch[3]) + jp, _ := strconv.Atoi(jmatch[3]) + + // weight each level of semver for comparison + iTotal := iM*marjorWeight + im*minorWeight + ip*patchWeight + jTotal := jM*marjorWeight + jm*minorWeight + jp*patchWeight + + // de-rank all prereleases by a major version + if imatch[4] != "" { + iTotal -= marjorWeight + } + if jmatch[4] != "" { + jTotal -= marjorWeight + } + + return iTotal > jTotal + }) +} + +func parsePkgName(childModFilePath string) (string, error) { + ss := strings.Split(childModFilePath, "/") + if len(ss) < 2 { + return "", fmt.Errorf("unable to parse package name from %q", childModFilePath) + } + return ss[len(ss)-1], nil +} + +type noopCloser struct { + w io.Writer +} + +func (n noopCloser) Write(p []byte) (int, error) { + return n.w.Write(p) +} + +func (n noopCloser) Close() error { return nil } + +func bumpSemverPatch(tag string) (string, error) { + splitTag := semverRegex.FindStringSubmatch(tag) + if len(splitTag) < 5 { + return "", fmt.Errorf("invalid tag layout: %q", tag) + } + var maj, min, pat int + var err error + if maj, err = strconv.Atoi(splitTag[1]); err != nil { + return "", fmt.Errorf("invalid tag layout: %q", tag) + } + if min, err = strconv.Atoi(splitTag[2]); err != nil { + return "", fmt.Errorf("invalid tag layout: %q", tag) + } + if pat, err = strconv.Atoi(splitTag[3]); err != nil { + return "", fmt.Errorf("invalid tag layout: %q", tag) + } + + if strings.Contains(tag, "/") { + splitTag := strings.Split(tag, "/") + return fmt.Sprintf("%s/v%d.%d.%d", strings.Join(splitTag[:len(splitTag)-1], "/"), maj, min, pat+1), nil + } + return fmt.Sprintf("v%d.%d.%d", maj, min, pat+1), nil +} + +// parseMetadata creates a mapping of potential modules to API full name. +func parseMetadata(r io.Reader) (map[string]string, error) { + m := map[string]struct { + Description string `json:"description"` + }{} + b, err := ioutil.ReadAll(r) + if err != nil { + return nil, err + } + if err := json.Unmarshal(b, &m); err != nil { + return nil, err + } + m2 := map[string]string{} + for k, v := range m { + k2 := k + if i := strings.Index(k2, "/apiv"); i > 0 { + k2 = k2[:i] + } + m2[k2] = v.Description + } + return m2, nil +} diff --git a/internal/carver/cmd/main_test.go b/internal/carver/cmd/main_test.go new file mode 100644 index 00000000000..6330fc65eea --- /dev/null +++ b/internal/carver/cmd/main_test.go @@ -0,0 +1,155 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and + +package main + +import ( + "strings" + "testing" + + "github.com/google/go-cmp/cmp" +) + +func TestBumpSemverPatch(t *testing.T) { + tests := []struct { + name string + in string + want string + wantErr bool + }{ + { + name: "full", + in: "v1.2.3", + want: "v1.2.4", + }, + { + name: "minor", + in: "v0.1.2", + want: "v0.1.3", + }, + { + name: "patch", + in: "v0.0.1", + want: "v0.0.2", + }, + { + name: "prefix", + in: "foo/v1.2.3", + want: "foo/v1.2.4", + }, + { + name: "longer prefix", + in: "foo/bar/v1.2.3", + want: "foo/bar/v1.2.4", + }, + { + name: "release candidate", + in: "v1.2.3-rc1", + want: "v1.2.4", + }, + { + name: "invalid input major", + in: "vs.0.1", + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := bumpSemverPatch(tt.in) + if tt.wantErr && err == nil { + t.Fatalf("bumpSemverPatch(%q) = nil, want error", tt.in) + } + if !tt.wantErr && err != nil { + t.Fatalf("bumpSemverPatch(%q) = %v, wantErr false", tt.in, err) + } + if got != tt.want { + t.Fatalf("bumpSemverPatch(%q) = %q, want %q", tt.in, got, tt.want) + } + }) + } +} + +func TestParsePkgName(t *testing.T) { + tests := []struct { + name string + in string + want string + wantErr bool + }{ + { + name: "valid", + in: "cloud.google.com/go/asset", + want: "asset", + }, + { + name: "valid", + in: "cloud.google.com/go/assdialogflow/cx", + want: "cx", + }, + { + name: "invalid", + in: "cloud.google.com", + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := parsePkgName(tt.in) + if tt.wantErr && err == nil { + t.Fatalf("parsePkgName(%q) = nil, want error", tt.in) + } + if !tt.wantErr && err != nil { + t.Fatalf("parsePkgName(%q) = %v, wantErr false", tt.in, err) + } + if got != tt.want { + t.Fatalf("parsePkgName(%q) = %q, want %q", tt.in, got, tt.want) + } + }) + } +} + +func TestSortTags(t *testing.T) { + in := []string{"v0.12.1", "v1.3.2", "v0.1.3-rc1", "v0.1.3", "v1.1.2-rc1", "v2.1.3-rc1"} + sortTags(in) + want := []string{"v1.3.2", "v2.1.3-rc1", "v0.12.1", "v0.1.3", "v1.1.2-rc1", "v0.1.3-rc1"} + if diff := cmp.Diff(want, in); diff != "" { + t.Errorf("sortTags() mismatch (-want +got):\n%s", diff) + } +} + +func TestParseMetadata(t *testing.T) { + data := `{ + "cloud.google.com/go/foo/apiv1": { + "description": "Foo API" + }, + "cloud.google.com/go/foo/bar/apiv1beta": { + "description": "FooBar API" + }, + "cloud.google.com/go/baz": { + "description": "Baz API" + } +}` + m, err := parseMetadata(strings.NewReader(data)) + if err != nil { + t.Fatalf("parseMetadata() = %v, want nil", err) + } + if key, want := "cloud.google.com/go/foo", "Foo API"; m[key] != want { + t.Fatalf("m[%q] = %q, want %q", key, m[key], want) + } + if key, want := "cloud.google.com/go/foo/bar", "FooBar API"; m[key] != want { + t.Fatalf("m[%q] = %q, want %q", key, m[key], want) + } + if key, want := "cloud.google.com/go/baz", "Baz API"; m[key] != want { + t.Fatalf("m[%q] = %q, want %q", key, m[key], want) + } +} diff --git a/internal/carver/go.mod b/internal/carver/go.mod new file mode 100644 index 00000000000..4ac636329a6 --- /dev/null +++ b/internal/carver/go.mod @@ -0,0 +1,5 @@ +module cloud.google.com/go/internal/carver + +go 1.16 + +require github.com/google/go-cmp v0.5.6 diff --git a/internal/carver/go.sum b/internal/carver/go.sum new file mode 100644 index 00000000000..03e1a9c43cf --- /dev/null +++ b/internal/carver/go.sum @@ -0,0 +1,4 @@ +github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=