chore: refactor genlocal into genbot (#3674)
Previously if you wanted to run genlocal you needed to have a
bunch of dependencies installed. Some of which can be a little
tricky to install correctly. These dependencies happen to be the
same as genbot. For this reason this commit refactors genlocal
into genbot so that the genbot in local mode can be run from
a docker container that has the exact same deps installed as our
nightly regen.
diff --git a/internal/gapicgen/cmd/genbot/Dockerfile b/internal/gapicgen/cmd/genbot/Dockerfile
index dfc8067..6ef6e7a 100644
--- a/internal/gapicgen/cmd/genbot/Dockerfile
+++ b/internal/gapicgen/cmd/genbot/Dockerfile
@@ -41,6 +41,11 @@
RUN echo -e '#!/bin/bash\n\
set -ex\n\
+ if [[ ${GENBOT_LOCAL_MODE} = "true" ]]; then\n\
+ cd internal/gapicgen\n\
+ go run cloud.google.com/go/internal/gapicgen/cmd/genbot\n\
+ exit 0\n\
+ fi\n\
go mod download \n\
go run cloud.google.com/go/internal/gapicgen/cmd/genbot \
' >> /run.sh
diff --git a/internal/gapicgen/cmd/genbot/README.md b/internal/gapicgen/cmd/genbot/README.md
index 0328f86..455fd5f 100644
--- a/internal/gapicgen/cmd/genbot/README.md
+++ b/internal/gapicgen/cmd/genbot/README.md
@@ -3,23 +3,38 @@
genbot is a binary for generating gapics and creating PRs with the results.
It is intended to be used as a bot, though it can be run locally too.
-## Getting certs
+## Prerequisites for running locally
-### Github
+Note that only step one, listed below, is required if you plan to run the code
+in docker.
-For Github, you need to generate/supply a Personal Access Token. More
-information on how that's done can be found here:
-[creating a personal access token](https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line).
+1. Clone this project: `git clone https://github.com/googleapis/google-cloud-go.git`
+1. Install [protoc](https://github.com/protocolbuffers/protobuf/releases)
+1. Install [Go](http://golang.org/dl)
+1. Add `$GOPATH/bin` to `PATH`
+1. Create a [GitHub access token](https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line).
+1. Install Go tools:
-## Running locally
+```bash
+go get \
+ github.com/golang/protobuf/protoc-gen-go \
+ golang.org/x/lint/golint \
+ golang.org/x/tools/cmd/goimports \
+ honnef.co/go/tools/cmd/staticcheck \
+ github.com/googleapis/gapic-generator-go/cmd/protoc-gen-go_gapic
+```
+
+## Generating code and PRs(bot mode)
+
+### Run genbot locally
Note: this may change your `~/.gitconfig`, `~/.gitcookies`, and use up
non-trivial amounts of space on your computer.
1. Make sure you are on a non-Windows platform. If you are using windows
continue on to the docker instructions.
-2. Make sure you have all the tools installed listed in genlocal's README.md
-3. Run:
+1. Make sure you have all the tools installed listed in genlocal's README.md
+1. Run:
```shell
cd /path/to/internal/gapicgen
@@ -30,7 +45,7 @@
--githubEmail=deklerk@google.com \
```
-## Run with docker
+### Run genbot with docker
Note: this can be quite slow (~10m).
@@ -41,12 +56,51 @@
cd /path/to/internal/gapicgen/cmd/genbot
docker build . -t genbot
docker run -t --rm --privileged \
- -v `pwd`/../..:/gapicgen \
- -e GITHUB_ACCESS_TOKEN \
- -e GITHUB_USERNAME \
- -e GITHUB_NAME \
- -e GITHUB_EMAIL \
- genbot
+ -v `pwd`/../..:/gapicgen \
+ -e GITHUB_ACCESS_TOKEN \
+ -e GITHUB_USERNAME \
+ -e GITHUB_NAME \
+ -e GITHUB_EMAIL \
+ genbot
+```
+
+## Generating code (local mode)
+
+Sometimes you may want to just generate gapic sources to test out
+new changes, test the generation of a new library, test new generator tweaks,
+run generators against googleapis-private, and various other local tasks. The
+local mode in genbot allows you to do just that.
+
+### Run genbot(local mode) locally
+
+```shell
+cd /path/to/internal/gapicgen
+go run cloud.google.com/go/internal/gapicgen/cmd/genbot \
+ -local \
+ -only-gapics \
+ -gocloud-dir=/path/to/google-cloud-go \
+ -gapic=cloud.google.com/go/foo/apiv1
+```
+
+### Run genbot(local mode) with docker
+
+```shell
+cd /path/to/internal/gapicgen
+docker build -t genbot -f cmd/genbot/Dockerfile .
+docker run --rm \
+ -v `pwd`/../..:/gapicgen \
+ -e GENBOT_LOCAL_MODE=true \
+ -e ONLY_GAPICS=true \
+ -e GOCLOUD_DIR=/gapicgen \
+ -e GAPIC_TO_GENERATE=cloud.google.com/go/foo/apiv1 \
+ genbot \
+```
+
+Note you can optionally mount in your Go module cache if you have Go installed.
+This will speed up the build a bit:
+
+```shell
+-v `go env GOMODCACHE`:/root/go/pkg/mod
```
## FAQ
diff --git a/internal/gapicgen/cmd/genbot/bot.go b/internal/gapicgen/cmd/genbot/bot.go
new file mode 100644
index 0000000..e5d8b8c
--- /dev/null
+++ b/internal/gapicgen/cmd/genbot/bot.go
@@ -0,0 +1,83 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !windows
+
+package main
+
+import (
+ "context"
+ "flag"
+ "fmt"
+ "log"
+ "time"
+)
+
+func genBot(ctx context.Context, githubAccessToken, githubUsername, githubName, githubEmail string) error {
+ for k, v := range map[string]string{
+ "githubAccessToken": githubAccessToken,
+ "githubUsername": githubUsername,
+ "githubName": githubName,
+ "githubEmail": githubEmail,
+ } {
+ if v == "" {
+ log.Printf("missing or empty value for required flag --%s\n", k)
+ flag.PrintDefaults()
+ }
+ }
+
+ // Setup the client and git environment.
+ githubClient, err := NewGithubClient(ctx, githubUsername, githubName, githubEmail, githubAccessToken)
+ if err != nil {
+ return err
+ }
+
+ // Check current regen status.
+ if pr, err := githubClient.GetRegenPR(ctx, "go-genproto", "open"); err != nil {
+ return err
+ } else if pr != nil {
+ return fmt.Errorf("There is already a re-generation in progress")
+ }
+ if pr, err := githubClient.GetRegenPR(ctx, "google-cloud-go", "open"); err != nil {
+ return err
+ } else if pr != nil {
+ if err := updateGocloudPR(ctx, githubClient, pr); err != nil {
+ return err
+ }
+ return nil
+ }
+ log.Println("checking if a pull request was already opened and merged today")
+ if pr, err := githubClient.GetRegenPR(ctx, "go-genproto", "closed"); err != nil {
+ return err
+ } else if pr != nil && hasCreatedPRToday(pr.Created) {
+ log.Println("skipping generation, already created and merged a go-genproto PR today")
+ return nil
+ }
+ if pr, err := githubClient.GetRegenPR(ctx, "google-cloud-go", "closed"); err != nil {
+ return err
+ } else if pr != nil && hasCreatedPRToday(pr.Created) {
+ log.Println("skipping generation, already created and merged a google-cloud-go PR today")
+ return nil
+ }
+
+ return generate(ctx, githubClient)
+}
+
+// hasCreatedPRToday checks if the created time of a PR is from today.
+func hasCreatedPRToday(created time.Time) bool {
+ now := time.Now()
+ today := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location())
+ log.Printf("Times -- Now: %v\tToday: %v\tPR Created: %v", now, today, created)
+ return created.After(today)
+}
diff --git a/internal/gapicgen/cmd/genbot/generate.go b/internal/gapicgen/cmd/genbot/generate.go
index b4b023c..154db21 100644
--- a/internal/gapicgen/cmd/genbot/generate.go
+++ b/internal/gapicgen/cmd/genbot/generate.go
@@ -53,16 +53,16 @@
grp, _ := errgroup.WithContext(ctx)
grp.Go(func() error {
- return gitClone("https://github.com/googleapis/googleapis", googleapisDir)
+ return gitDeepClone("https://github.com/googleapis/googleapis", googleapisDir)
})
grp.Go(func() error {
- return gitClone("https://github.com/googleapis/go-genproto", genprotoDir)
+ return gitDeepClone("https://github.com/googleapis/go-genproto", genprotoDir)
})
grp.Go(func() error {
- return gitClone("https://github.com/googleapis/google-cloud-go", gocloudDir)
+ return gitDeepClone("https://github.com/googleapis/google-cloud-go", gocloudDir)
})
grp.Go(func() error {
- return gitClone("https://github.com/protocolbuffers/protobuf", protoDir)
+ return gitDeepClone("https://github.com/protocolbuffers/protobuf", protoDir)
})
if err := grp.Wait(); err != nil {
log.Println(err)
@@ -140,7 +140,7 @@
}
// gitClone clones a repository in the given directory.
-func gitClone(repo, dir string) error {
+func gitDeepClone(repo, dir string) error {
log.Printf("cloning %s\n", repo)
_, err := git.PlainClone(dir, false, &git.CloneOptions{
diff --git a/internal/gapicgen/cmd/genbot/github.go b/internal/gapicgen/cmd/genbot/github.go
index c84e04d..24b296d 100644
--- a/internal/gapicgen/cmd/genbot/github.go
+++ b/internal/gapicgen/cmd/genbot/github.go
@@ -235,7 +235,7 @@
// Can't assign the submitter of the PR as a reviewer.
var reviewers []string
for _, r := range githubReviewers {
- if r != *githubUsername {
+ if r != gc.Username {
reviewers = append(reviewers, r)
}
}
diff --git a/internal/gapicgen/cmd/genbot/local.go b/internal/gapicgen/cmd/genbot/local.go
new file mode 100644
index 0000000..a45b5d6
--- /dev/null
+++ b/internal/gapicgen/cmd/genbot/local.go
@@ -0,0 +1,104 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !windows
+
+package main
+
+import (
+ "context"
+ "io/ioutil"
+ "log"
+ "os"
+ "path/filepath"
+
+ "cloud.google.com/go/internal/gapicgen/generator"
+ "golang.org/x/sync/errgroup"
+ "gopkg.in/src-d/go-git.v4"
+)
+
+type localConfig struct {
+ googleapisDir string
+ gocloudDir string
+ genprotoDir string
+ protoDir string
+ gapicToGenerate string
+ onlyGapics bool
+}
+
+func genLocal(ctx context.Context, c localConfig) error {
+ log.Println("creating temp dir")
+ tmpDir, err := ioutil.TempDir("", "update-genproto")
+ if err != nil {
+ log.Fatal(err)
+ }
+ log.Printf("temp dir created at %s\n", tmpDir)
+ tmpGoogleapisDir := filepath.Join(tmpDir, "googleapis")
+ tmpGenprotoDir := filepath.Join(tmpDir, "genproto")
+ tmpGocloudDir := filepath.Join(tmpDir, "gocloud")
+ tmpProtoDir := filepath.Join(tmpDir, "proto")
+
+ // Clone repositories if needed.
+ grp, _ := errgroup.WithContext(ctx)
+ gitShallowClone(grp, "https://github.com/googleapis/googleapis.git", c.googleapisDir, tmpGoogleapisDir)
+ gitShallowClone(grp, "https://github.com/googleapis/go-genproto", c.genprotoDir, tmpGenprotoDir)
+ gitShallowClone(grp, "https://github.com/googleapis/google-cloud-go", c.gocloudDir, tmpGocloudDir)
+ gitShallowClone(grp, "https://github.com/protocolbuffers/protobuf", c.protoDir, tmpProtoDir)
+ if err := grp.Wait(); err != nil {
+ log.Println(err)
+ }
+
+ // Regen.
+ conf := &generator.Config{
+ GoogleapisDir: deafultDir(tmpGoogleapisDir, c.googleapisDir),
+ GenprotoDir: deafultDir(tmpGenprotoDir, c.genprotoDir),
+ GapicDir: deafultDir(tmpGocloudDir, c.gocloudDir),
+ ProtoDir: deafultDir(tmpProtoDir, c.protoDir),
+ GapicToGenerate: c.gapicToGenerate,
+ OnlyGenerateGapic: c.onlyGapics,
+ LocalMode: true,
+ }
+ if _, err := generator.Generate(ctx, conf); err != nil {
+ log.Printf("Generator ran (and failed) in %s\n", tmpDir)
+ log.Fatal(err)
+ }
+ return nil
+}
+
+// gitShallowClone clones a repository into the provided tmpDir if a dir has not
+// been specified.
+func gitShallowClone(eg *errgroup.Group, repo, dir, tmpDir string) {
+ if dir != "" {
+ return
+ }
+ eg.Go(func() error {
+ log.Printf("cloning %s\n", repo)
+
+ _, err := git.PlainClone(tmpDir, false, &git.CloneOptions{
+ URL: repo,
+ Progress: os.Stdout,
+ Depth: 1,
+ Tags: git.NoTags,
+ })
+ return err
+ })
+}
+
+// deafultDir returns the default option if dir is not set.
+func deafultDir(def, dir string) string {
+ if dir == "" {
+ return def
+ }
+ return dir
+}
diff --git a/internal/gapicgen/cmd/genbot/main.go b/internal/gapicgen/cmd/genbot/main.go
index bf07d2f..aa598b7 100644
--- a/internal/gapicgen/cmd/genbot/main.go
+++ b/internal/gapicgen/cmd/genbot/main.go
@@ -21,44 +21,13 @@
import (
"context"
"flag"
- "fmt"
"log"
"os"
- "time"
+ "strconv"
"cloud.google.com/go/internal/gapicgen"
)
-var (
- toolsNeeded = []string{"git", "go", "protoc"}
-
- githubAccessToken = flag.String("githubAccessToken", "", "Get an access token at https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line")
- githubUsername = flag.String("githubUsername", "", "ex -githubUsername=jadekler")
- githubName = flag.String("githubName", "", "ex -githubName=\"Jean de Klerk\"")
- githubEmail = flag.String("githubEmail", "", "ex -githubEmail=deklerk@google.com")
-
- usage = func() {
- fmt.Fprintln(os.Stderr, `genbot \
- -githubAccessToken=11223344556677889900aabbccddeeff11223344 \
- -githubUsername=jadekler \
- -githubEmail=deklerk@google.com \
- -githubName="Jean de Klerk" \
-
--githubAccessToken
- The access token to authenticate to github. Get this at https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line.
-
--githubUsername
- The username to use in the github commit.
-
--githubName
- The name to use in the github commit.
-
--githubEmail
- The email to use in the github commit.`)
- os.Exit(2)
- }
-)
-
type prStatus uint8
func (ps prStatus) Has(status prStatus) bool { return ps&status != 0 }
@@ -71,83 +40,48 @@
func main() {
log.SetFlags(0)
-
- flag.Usage = usage
- flag.Parse()
- if *githubAccessToken == "" {
- *githubAccessToken = os.Getenv("GITHUB_ACCESS_TOKEN")
+ if err := gapicgen.VerifyAllToolsExist([]string{"git", "go", "protoc"}); err != nil {
+ log.Fatal(err)
}
- if *githubUsername == "" {
- *githubUsername = os.Getenv("GITHUB_USERNAME")
- }
- if *githubName == "" {
- *githubName = os.Getenv("GITHUB_NAME")
- }
- if *githubEmail == "" {
- *githubEmail = os.Getenv("GITHUB_EMAIL")
- }
-
- for k, v := range map[string]string{
- "githubAccessToken": *githubAccessToken,
- "githubUsername": *githubUsername,
- "githubName": *githubName,
- "githubEmail": *githubEmail,
- } {
- if v == "" {
- log.Printf("missing or empty value for required flag --%s\n", k)
- usage()
- }
- }
-
ctx := context.Background()
- // Setup the client and git environment.
- if err := gapicgen.VerifyAllToolsExist(toolsNeeded); err != nil {
- log.Fatal(err)
- }
- githubClient, err := NewGithubClient(ctx, *githubUsername, *githubName, *githubEmail, *githubAccessToken)
- if err != nil {
- log.Fatal(err)
- }
+ // General bot flags
+ githubAccessToken := flag.String("githubAccessToken", os.Getenv("GITHUB_ACCESS_TOKEN"), "The token used to open pull requests.")
+ githubUsername := flag.String("githubUsername", os.Getenv("GITHUB_USERNAME"), "The GitHub user name for the author.")
+ githubName := flag.String("githubName", os.Getenv("GITHUB_NAME"), "The name of the author for git commits.")
+ githubEmail := flag.String("githubEmail", os.Getenv("GITHUB_EMAIL"), "The email address of the author.")
+ localMode := flag.Bool("local", strToBool(os.Getenv("GENBOT_LOCAL_MODE")), "Enables generating sources locally. This mode will not open any pull requests.")
- // Check current regen status.
- if pr, err := githubClient.GetRegenPR(ctx, "go-genproto", "open"); err != nil {
- log.Fatal(err)
- } else if pr != nil {
- log.Println("There is already a re-generation in progress")
- return
- }
- if pr, err := githubClient.GetRegenPR(ctx, "google-cloud-go", "open"); err != nil {
- log.Fatal(err)
- } else if pr != nil {
- if err := updateGocloudPR(ctx, githubClient, pr); err != nil {
+ // flags for local mode
+ googleapisDir := flag.String("googleapis-dir", os.Getenv("GOOGLEAPIS_DIR"), "Directory where sources of googleapis/googleapis resides. If unset the sources will be cloned to a temporary directory that is not cleaned up.")
+ gocloudDir := flag.String("gocloud-dir", os.Getenv("GOCLOUD_DIR"), "Directory where sources of googleapis/google-cloud-go resides. If unset the sources will be cloned to a temporary directory that is not cleaned up.")
+ genprotoDir := flag.String("genproto-dir", os.Getenv("GENPROTO_DIR"), "Directory where sources of googleapis/go-genproto resides. If unset the sources will be cloned to a temporary directory that is not cleaned up.")
+ protoDir := flag.String("proto-dir", os.Getenv("PROTO_DIR"), "Directory where sources of google/protobuf resides. If unset the sources will be cloned to a temporary directory that is not cleaned up.")
+ gapicToGenerate := flag.String("gapic", os.Getenv("GAPIC_TO_GENERATE"), `Specifies which gapic to generate. The value should be in the form of an import path (Ex: cloud.google.com/go/pubsub/apiv1). The default "" generates all gapics.`)
+ onlyGapics := flag.Bool("only-gapics", strToBool(os.Getenv("ONLY_GAPICS")), "Enabling stops regenerating genproto.")
+
+ flag.Parse()
+
+ if *localMode {
+ if err := genLocal(ctx, localConfig{
+ googleapisDir: *googleapisDir,
+ gocloudDir: *gocloudDir,
+ genprotoDir: *genprotoDir,
+ protoDir: *protoDir,
+ gapicToGenerate: *gapicToGenerate,
+ onlyGapics: *onlyGapics,
+ }); err != nil {
log.Fatal(err)
}
return
}
- log.Println("checking if a pull request was already opened and merged today")
- if pr, err := githubClient.GetRegenPR(ctx, "go-genproto", "closed"); err != nil {
- log.Fatal(err)
- } else if pr != nil && hasCreatedPRToday(pr.Created) {
- log.Println("skipping generation, already created and merged a go-genproto PR today")
- return
- }
- if pr, err := githubClient.GetRegenPR(ctx, "google-cloud-go", "closed"); err != nil {
- log.Fatal(err)
- } else if pr != nil && hasCreatedPRToday(pr.Created) {
- log.Println("skipping generation, already created and merged a google-cloud-go PR today")
- return
- }
-
- if err := generate(ctx, githubClient); err != nil {
+ if err := genBot(ctx, *githubAccessToken, *githubUsername, *githubName, *githubEmail); err != nil {
log.Fatal(err)
}
}
-// hasCreatedPRToday checks if the created time of a PR is from today.
-func hasCreatedPRToday(created time.Time) bool {
- now := time.Now()
- today := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location())
- log.Printf("Times -- Now: %v\tToday: %v\tPR Created: %v", now, today, created)
- return created.After(today)
+func strToBool(s string) bool {
+ // Treat error as false
+ b, _ := strconv.ParseBool(s)
+ return b
}
diff --git a/internal/gapicgen/cmd/genlocal/README.md b/internal/gapicgen/cmd/genlocal/README.md
deleted file mode 100644
index 3532ee4..0000000
--- a/internal/gapicgen/cmd/genlocal/README.md
+++ /dev/null
@@ -1,34 +0,0 @@
-# genlocal
-
-genlocal is a binary for generating gapics locally. It may be used to test out
-new changes, test the generation of a new library, test new generator tweaks,
-run generators against googleapis-private, and various other local tasks.
-
-## Required tools
-
-*Note*: If you are on a Windows platform you will need to install these tools
-in a linux docker container: [Install docker](https://www.docker.com/get-started)
-
-1. Install [protoc](https://github.com/protocolbuffers/protobuf/releases)
-2. Install [Go](http://golang.org/dl)
-3. Install python3, pip3
-4. Install virtualenv `pip3 install virtualenv`
-5. Install Go tools:
-
-```bash
-go get \
- github.com/golang/protobuf/protoc-gen-go \
- golang.org/x/lint/golint \
- golang.org/x/tools/cmd/goimports \
- honnef.co/go/tools/cmd/staticcheck \
- github.com/googleapis/gapic-generator-go/cmd/protoc-gen-go_gapic
-```
-
-## Running
-
-`git clone` this project if you don't already have it checked-out locally.
-
-```bash
-cd /path/to/google-cloud-go/internal/gapicgen
-go run cloud.google.com/go/internal/gapicgen/cmd/genlocal
-```
diff --git a/internal/gapicgen/cmd/genlocal/main.go b/internal/gapicgen/cmd/genlocal/main.go
deleted file mode 100644
index 0f004a0..0000000
--- a/internal/gapicgen/cmd/genlocal/main.go
+++ /dev/null
@@ -1,113 +0,0 @@
-// Copyright 2019 Google LLC
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// +build !windows
-
-// genlocal is a binary for generating gapics locally. It may be used to test out
-// new changes, test the generation of a new library, test new generator tweaks,
-// run generators against googleapis-private, and various other local tasks.
-package main
-
-import (
- "context"
- "flag"
- "io/ioutil"
- "log"
- "os"
- "path/filepath"
- "strings"
-
- "cloud.google.com/go/internal/gapicgen"
- "cloud.google.com/go/internal/gapicgen/generator"
- "golang.org/x/sync/errgroup"
- "gopkg.in/src-d/go-git.v4"
-)
-
-var (
- toolsNeeded = []string{"go", "protoc"}
-)
-
-func main() {
- log.SetFlags(0)
- if err := gapicgen.VerifyAllToolsExist(toolsNeeded); err != nil {
- log.Fatal(err)
- }
-
- log.Println("creating temp dir")
- tmpDir, err := ioutil.TempDir("", "update-genproto")
- if err != nil {
- log.Fatal(err)
- }
- log.Printf("temp dir created at %s\n", tmpDir)
-
- googleapisDir := flag.String("googleapis-dir", filepath.Join(tmpDir, "googleapis"), "Directory where sources of googleapis/googleapis resides. If unset the sources will be cloned to a temporary directory that is not cleaned up.")
- gocloudDir := flag.String("gocloud-dir", filepath.Join(tmpDir, "gocloud"), "Directory where sources of googleapis/google-cloud-go resides. If unset the sources will be cloned to a temporary directory that is not cleaned up.")
- genprotoDir := flag.String("genproto-dir", filepath.Join(tmpDir, "genproto"), "Directory where sources of googleapis/go-genproto resides. If unset the sources will be cloned to a temporary directory that is not cleaned up.")
- protoDir := flag.String("proto-dir", filepath.Join(tmpDir, "proto"), "Directory where sources of google/protobuf resides. If unset the sources will be cloned to a temporary directory that is not cleaned up.")
- gapicToGenerate := flag.String("gapic", "", `Specifies which gapic to generate. The value should be in the form of an import path (Ex: cloud.google.com/go/pubsub/apiv1). The default "" generates all gapics.`)
- onlyGapics := flag.Bool("only-gapics", false, "Enabling stops regenerating genproto.")
- verbose := flag.Bool("verbose", false, "Enables verbose logging.")
- flag.Parse()
-
- ctx := context.Background()
-
- // Clone repositories if needed.
- grp, _ := errgroup.WithContext(ctx)
- gitClone(grp, "https://github.com/googleapis/googleapis.git", *googleapisDir, tmpDir)
- gitClone(grp, "https://github.com/googleapis/go-genproto", *genprotoDir, tmpDir)
- gitClone(grp, "https://github.com/googleapis/google-cloud-go", *gocloudDir, tmpDir)
- gitClone(grp, "https://github.com/protocolbuffers/protobuf", *protoDir, tmpDir)
- if err := grp.Wait(); err != nil {
- log.Println(err)
- }
-
- // Regen.
- conf := &generator.Config{
- GoogleapisDir: *googleapisDir,
- GenprotoDir: *genprotoDir,
- GapicDir: *gocloudDir,
- ProtoDir: *protoDir,
- GapicToGenerate: *gapicToGenerate,
- OnlyGenerateGapic: *onlyGapics,
- }
- changes, err := generator.Generate(ctx, conf)
- if err != nil {
- log.Printf("Generator ran (and failed) in %s\n", tmpDir)
- log.Fatal(err)
- }
-
- // Log results.
- log.Println(genprotoDir)
- log.Println(gocloudDir)
-
- if *verbose {
- log.Println(generator.FormatChanges(changes, false))
- }
-}
-
-// gitClone clones a repository in the given directory if dir is not in tmpDir.
-func gitClone(eg *errgroup.Group, repo, dir, tmpDir string) {
- if !strings.HasPrefix(dir, tmpDir) {
- return
- }
- eg.Go(func() error {
- log.Printf("cloning %s\n", repo)
-
- _, err := git.PlainClone(dir, false, &git.CloneOptions{
- URL: repo,
- Progress: os.Stdout,
- })
- return err
- })
-}
diff --git a/internal/gapicgen/generator/generator.go b/internal/gapicgen/generator/generator.go
index f5b7f97..c3831e4 100644
--- a/internal/gapicgen/generator/generator.go
+++ b/internal/gapicgen/generator/generator.go
@@ -36,6 +36,7 @@
ProtoDir string
GapicToGenerate string
OnlyGenerateGapic bool
+ LocalMode bool
}
// Generate generates genproto and gapics.
@@ -51,13 +52,16 @@
return nil, fmt.Errorf("error generating gapics (may need to check logs for more errors): %v", err)
}
- changes, err := gatherChanges(conf.GoogleapisDir, conf.GenprotoDir)
- if err != nil {
- return nil, fmt.Errorf("error gathering commit info")
- }
-
- if err := recordGoogleapisHash(conf.GoogleapisDir, conf.GenprotoDir); err != nil {
- return nil, err
+ var changes []*ChangeInfo
+ if !conf.LocalMode {
+ var err error
+ changes, err = gatherChanges(conf.GoogleapisDir, conf.GenprotoDir)
+ if err != nil {
+ return nil, fmt.Errorf("error gathering commit info")
+ }
+ if err := recordGoogleapisHash(conf.GoogleapisDir, conf.GenprotoDir); err != nil {
+ return nil, err
+ }
}
return changes, nil
