autogogen: add auto-generator gapics/protos
Design: go/gapicgen-v2
This adds three binaries:
- internal/gapicgen/cmd/genlocal: Generates genproto+gapics locally. Intended to
be run by humans - for example, when testing new changes, or adding a new
gapic, or generating from googleapis-private.
- internal/gapicgen/cmd/genbot: Generates genproto+gapics locally, and creates
CLs/PRs for them and assigns to the appropriate folks. Intended to be run
periodically as a bot.
- internal/gapicgen/cmd/genmgr: Checks for an outstanding gapic regen CL
that needs to have reviewers added and go.mod update, and then does so.
Intended to be run periodically as a bot.
This change obsoletes:
- genproto/regen.go
- genproto/regen.sh
- gocloud/gapics.txt
- gocloud/manuals.txt
- gocloud/microgens.csv
- gocloud/tidyall.sh
- gocloud/regen-gapics.sh
Change-Id: I36b4461cc0df00cb43cb7fd150470781e25c77f2
Reviewed-on: https://code-review.googlesource.com/c/gocloud/+/48072
Reviewed-by: kokoro <noreply+kokoro@google.com>
Reviewed-by: Tyler Bui-Palsulich <tbp@google.com>
Reviewed-by: Noah Dietz <ndietz@google.com>
Reviewed-by: Cody Oss <codyoss@google.com>
diff --git a/gapics.txt b/gapics.txt
deleted file mode 100644
index 323ffbd..0000000
--- a/gapics.txt
+++ /dev/null
@@ -1,38 +0,0 @@
-google/api/expr/artman_cel.yaml
-google/cloud/asset/artman_cloudasset_v1beta1.yaml
-google/cloud/asset/artman_cloudasset_v1p2beta1.yaml
-google/iam/credentials/artman_iamcredentials_v1.yaml
-google/cloud/automl/artman_automl_v1.yaml
-google/cloud/automl/artman_automl_v1beta1.yaml
-google/cloud/dataproc/artman_dataproc_v1.yaml
-google/cloud/dataproc/artman_dataproc_v1beta2.yaml
-google/cloud/dialogflow/v2/artman_dialogflow_v2.yaml
-google/cloud/irm/artman_irm_v1alpha2.yaml
-google/cloud/kms/artman_cloudkms.yaml
-google/cloud/language/artman_language_v1beta2.yaml
-google/cloud/oslogin/artman_oslogin_v1.yaml
-google/cloud/oslogin/artman_oslogin_v1beta.yaml
-google/cloud/recaptchaenterprise/artman_recaptchaenterprise_v1beta1.yaml
-google/cloud/redis/artman_redis_v1beta1.yaml
-google/cloud/redis/artman_redis_v1.yaml
-google/cloud/securitycenter/artman_securitycenter_v1beta1.yaml
-google/cloud/securitycenter/artman_securitycenter_v1.yaml
-google/cloud/talent/artman_talent_v4beta1.yaml
-google/cloud/videointelligence/artman_videointelligence_v1beta2.yaml
-google/cloud/vision/artman_vision_v1p1beta1.yaml
-google/devtools/artman_clouddebugger.yaml
-google/devtools/cloudbuild/artman_cloudbuild.yaml
-google/devtools/clouderrorreporting/artman_errorreporting.yaml
-google/devtools/cloudtrace/artman_cloudtrace_v1.yaml
-google/devtools/cloudtrace/artman_cloudtrace_v2.yaml
-google/devtools/containeranalysis/artman_containeranalysis_v1beta1.yaml
-google/firestore/artman_firestore.yaml
-google/firestore/admin/artman_firestore_v1.yaml
-google/logging/artman_logging.yaml
-google/longrunning/artman_longrunning.yaml
-google/monitoring/artman_monitoring.yaml
-google/privacy/dlp/artman_dlp_v2.yaml
-google/pubsub/artman_pubsub.yaml
-google/spanner/admin/database/artman_spanner_admin_database.yaml
-google/spanner/admin/instance/artman_spanner_admin_instance.yaml
-google/spanner/artman_spanner.yaml
diff --git a/go.mod b/go.mod
index 053559a..7696d42 100644
--- a/go.mod
+++ b/go.mod
@@ -17,8 +17,9 @@
go.opencensus.io v0.22.0
golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136
golang.org/x/lint v0.0.0-20190930215403-16217165b5de
- golang.org/x/net v0.0.0-20190620200207-3b0461eec859
+ golang.org/x/net v0.0.0-20190724013045-ca1201d0de80
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
+ golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e // indirect
golang.org/x/text v0.3.2
golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2
google.golang.org/api v0.14.0
diff --git a/go.sum b/go.sum
index 1cc173d..e79c04e 100644
--- a/go.sum
+++ b/go.sum
@@ -57,8 +57,10 @@
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024 h1:rBMNdlhTLzJjJSDIjNEXX1Pz3Hmwmz91v+zycvx9PJc=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
go.opencensus.io v0.21.0 h1:mU6zScU4U1YAFPHEHYk+3JC4SY7JxgkqS10ZOSyksNg=
@@ -107,6 +109,8 @@
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 h1:Ao/3l156eZf2AW5wK8a7/smtodRU+gha3+BeqJ69lRk=
+golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 h1:Wo7BWFiOk0QRFMLYMqJGFMd9CgUAcGx7V+qEg/h5IBI=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -130,6 +134,8 @@
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0 h1:HyfiK1WMnHj5FXFXatD+Qs1A/xC2Run6RzeW1SyHxpc=
golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e h1:D5TXcfTk7xF7hvieo4QErS3qqCB4teTffacDWr7CI+0=
+golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2 h1:z99zHgr7hKfrUcX/KsoJk5FJfjTceCKIp96+biqP4To=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -196,6 +202,7 @@
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
google.golang.org/grpc v1.21.1 h1:j6XxA85m/6txkUCHvzlV5f+HBNl/1r5cZ2A/3IEFOO8=
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
diff --git a/internal/gapicgen/README.md b/internal/gapicgen/README.md
new file mode 100644
index 0000000..1e696fa
--- /dev/null
+++ b/internal/gapicgen/README.md
@@ -0,0 +1,16 @@
+# gapicgen
+
+gapicgen contains three binaries:
+
+- `cloud.google.com/go/internal/gapicgen/cmd/genlocal`: Generates
+ genproto+gapics locally. Intended to be run by humans - for example, when
+ testing new changes, or adding a new gapic, or generating from
+ googleapis-private.
+- `cloud.google.com/go/internal/gapicgen/cmd/genbot`: Generates genproto+gapics
+ locally, and creates CLs/PRs for them and assigns to the appropriate folks.
+ Intended to be run periodically as a bot, but humans can use it too.
+- `cloud.google.com/go/internal/gapicgen/cmd/genmgr`: Checks for an outstanding
+ gapic regen CL that needs to have reviewers added and go.mod update, and then
+ does so. Intended to be run periodically as a bot, but humans can use it too.
+
+See the README.md in each folder for more specific instructions.
\ No newline at end of file
diff --git a/internal/gapicgen/cmd/genbot/Dockerfile b/internal/gapicgen/cmd/genbot/Dockerfile
new file mode 100644
index 0000000..805c31c
--- /dev/null
+++ b/internal/gapicgen/cmd/genbot/Dockerfile
@@ -0,0 +1,93 @@
+# Build protoc. Source: https://gist.github.com/rizo/513849f35178d19a13adcddf2d045a19.
+FROM golang:1.8.3-alpine3.6 as protoc-builder
+
+# System setup
+RUN apk update && apk add git curl build-base autoconf automake libtool
+
+# Install protoc
+ENV PROTOBUF_URL https://github.com/google/protobuf/releases/download/v3.6.1/protobuf-cpp-3.6.1.tar.gz
+RUN curl -L -o /tmp/protobuf.tar.gz $PROTOBUF_URL
+WORKDIR /tmp/
+RUN tar xvzf protobuf.tar.gz
+WORKDIR /tmp/protobuf-3.6.1
+RUN mkdir /export
+RUN ./autogen.sh && \
+ ./configure --prefix=/export && \
+ make -j 3 && \
+ make check && \
+ make install
+
+# Export dependencies
+RUN cp /usr/lib/libstdc++* /export/lib/
+RUN cp /usr/lib/libgcc_s* /export/lib/
+
+FROM docker:stable-dind
+
+RUN apk update && \
+ apk add ca-certificates wget git unzip
+
+# Install Go.
+RUN cd $(mktemp -d) && \
+ wget https://dl.google.com/go/go1.13.4.linux-amd64.tar.gz && \
+ tar xvzf go1.13.4.linux-amd64.tar.gz && \
+ mv go /go
+ENV PATH="${PATH}:/go/bin"
+# Source: https://stackoverflow.com/questions/34729748/installed-go-binary-not-found-in-path-on-alpine-linux-docker
+RUN mkdir /lib64 && ln -s /lib/libc.musl-x86_64.so.1 /lib64/ld-linux-x86-64.so.2
+RUN go version
+
+# Install python and pip.
+RUN apk add python python3 py-pip py3-pip py-virtualenv py3-virtualenv python2-dev python3-dev libffi-dev
+RUN python --version
+RUN python3 --version
+RUN pip --version
+RUN pip3 --version
+RUN virtualenv --version
+
+# Install protoc.
+COPY --from=protoc-builder /export /usr
+RUN protoc --version
+
+# Install Go tools.
+RUN go get \
+ github.com/golang/protobuf/protoc-gen-go \
+ golang.org/x/lint/golint \
+ golang.org/x/tools/cmd/goimports \
+ honnef.co/go/tools/cmd/staticcheck \
+ github.com/golang/protobuf/protoc-gen-go \
+ github.com/googleapis/gapic-generator-go/cmd/protoc-gen-go_gapic \
+ golang.org/x/review/git-codereview
+ENV PATH="${PATH}:/root/go/bin"
+
+# Install bash and ssh tools (needed to run regen.sh etc).
+RUN apk add bash openssh openssh-client
+# Source: http://debuggable.com/posts/disable-strict-host-checking-for-git-clone:49896ff3-0ac0-4263-9703-1eae4834cda3
+RUN mkdir /root/.ssh && echo -e "Host github.com\n\tStrictHostKeyChecking no\n" >> /root/.ssh/config
+RUN which bash
+# Install tools necessary for artman.
+RUN apk add alpine-sdk build-base openssl-dev
+RUN which gcc
+RUN pip install googleapis-artman
+RUN which artman
+
+RUN echo -e '#!/bin/bash\n\
+set -ex\n\
+dockerd-entrypoint.sh &\n\
+unset DOCKER_HOST\n\
+go mod download & # download deps whilst we wait for dockerd to start \n\
+sleep 15 # wait for docker to start\n\
+docker ps\n\
+docker pull googleapis/artman:latest & # download latest artman whilst we wait for genbot to start \n\
+go run cloud.google.com/go/internal/gapicgen/cmd/genbot \
+ --accessToken=$ACCESS_TOKEN \
+ --githubUsername=$GITHUB_USERNAME \
+ --githubName="$GITHUB_NAME" \
+ --githubEmail=$GITHUB_EMAIL \
+ --githubSSHKeyPath=$GITHUB_SSH_KEY_PATH \
+ --gerritCookieName=$GERRIT_COOKIE_NAME \
+ --gerritCookieValue=$GERRIT_COOKIE_VALUE \n\
+' >> /run.sh
+RUN chmod a+x /run.sh
+
+WORKDIR /gapicgen
+CMD ["bash", "-c", "/run.sh"]
\ No newline at end of file
diff --git a/internal/gapicgen/cmd/genbot/README.md b/internal/gapicgen/cmd/genbot/README.md
new file mode 100644
index 0000000..93f8c5e
--- /dev/null
+++ b/internal/gapicgen/cmd/genbot/README.md
@@ -0,0 +1,53 @@
+# genbot
+
+genbot is a binary for generating gapics and creating CLs/PRs with the results.
+It is intended to be used as a bot, though it can be run locally too.
+
+## Getting certs
+
+1. Grab github personal access token (see: https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line)
+1. Grab a gerrit HTTP auth cookie at https://code-review.googlesource.com/settings/#HTTPCredentials > Obtain password > `git-your@email.com=SomeHash....`
+
+## Running locally
+
+Note: this may change your ~/.gitconfig, ~/.gitcookies, and use up non-trivial
+amounts of space on your computer.
+
+1. Make sure you have all the tools installed listed in genlocal's README.md
+1. Create a fork of genproto for whichever github user you're going to be
+ running as: https://github.com/googleapis/go-genproto/
+1. Run:
+
+```
+go run cloud.google.com/go/internal/gapicgen/genbot \
+ --accessToken=11223344556677889900aabbccddeeff11223344 \
+ --githubUsername=jadekler \
+ --githubName="Jean de Klerk" \
+ --githubEmail=deklerk@google.com \
+ --githubSSHKeyPath=/path/to/.ssh/github_rsa \
+ --gerritCookieName=o \
+ --gerritCookieValue=<cookie>
+```
+
+## Run with docker
+
+Note: this can be quite slow (~10m).
+
+Note: this may leave a lot of docker resources laying around. Use
+`docker system prune` to clean up after runs.
+
+```
+cd /path/to/internal/gapicgen/cmd/genbot
+docker build . -t genbot
+docker run -t --rm --privileged \
+ -v `pwd`/../..:/gapicgen \
+ -v /path/to/your/ssh/key/directory:/.ssh \
+ -e "ACCESS_TOKEN=11223344556677889900aabbccddeeff11223344" \
+ -e "GITHUB_USERNAME=jadekler" \
+ -e "GITHUB_NAME=\"Jean de Klerk\"" \
+ -e "GITHUB_EMAIL=deklerk@google.com" \
+ -e "GITHUB_SSH_KEY_PATH=/.ssh/name_of_your_github_rsa_file" \
+ -e "GERRIT_COOKIE_NAME=o" \
+ -e "GERRIT_COOKIE_VALUE=<cookie>" \
+ genbot
+```
diff --git a/internal/gapicgen/cmd/genbot/gapics.go b/internal/gapicgen/cmd/genbot/gapics.go
new file mode 100644
index 0000000..4e8ddd1
--- /dev/null
+++ b/internal/gapicgen/cmd/genbot/gapics.go
@@ -0,0 +1,97 @@
+// Copyright 2019 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "log"
+ "os"
+ "os/exec"
+ "regexp"
+)
+
+// gerritRegex is used to find gerrit links.
+var gerritRegex = regexp.MustCompile(`https://code-review.googlesource.com.+[0-9]+`)
+
+const (
+ gerritCommitTitle = "all: auto-regenerate gapics"
+ gerritCommitBody = `
+This is an auto-generated regeneration of the gapic clients by autogogen. Once
+the corresponding genproto PR is submitted, autotogen will update this CL with
+a newer dependency to the newer version of genproto and assign reviewers to
+this CL.
+
+If you have been assigned to review this CL, please:
+
+- Ensure that the version of genproto in go.mod has been updated.
+- Ensure that CI is passing. If it's failing, it requires your manual attention.
+- +2 and submit this CL if you believe it's ready to ship.
+`
+)
+
+// clGocloud creates a CL for the given gocloud change (including a link to
+// the given genproto PR).
+func clGocloud(ctx context.Context, gocloudDir string, genprotoPRNum int) (url string, _ error) {
+ log.Println("creating gocloud CL")
+
+ newBody := fmt.Sprintf(`%s
+
+Corresponding genproto PR: https://github.com/googleapis/go-genproto/pull/%d
+`, gerritCommitBody, genprotoPRNum)
+
+ // Write command output to both os.Stderr and local, so that we can check
+ // for gerrit URL.
+ inmem := bytes.NewBuffer([]byte{}) // TODO(deklerk): Try `var inmem bytes.Buffer`.
+ w := io.MultiWriter(os.Stderr, inmem)
+
+ c := exec.Command("/bin/bash", "-c", `
+set -ex
+
+CHANGE_ID=$(echo $RANDOM | git hash-object --stdin)
+git checkout master
+git branch -d regen_gapics || true
+git add -A
+git checkout -b regen_gapics
+git commit -m "$COMMIT_TITLE" -m "$COMMIT_BODY" -m "Change-Id: I$CHANGE_ID"
+git-codereview mail
+`)
+ c.Stdout = os.Stdout
+ c.Stderr = w
+ c.Stdin = os.Stdin // Prevents "the input device is not a TTY" error.
+ c.Env = []string{
+ fmt.Sprintf("COMMIT_TITLE=%s", gerritCommitTitle),
+ fmt.Sprintf("COMMIT_BODY=%s", newBody),
+ fmt.Sprintf("PATH=%s", os.Getenv("PATH")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands.
+ fmt.Sprintf("HOME=%s", os.Getenv("HOME")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands.
+ }
+ c.Dir = gocloudDir
+ if err := c.Run(); err != nil {
+ return "", err
+ }
+
+ b := inmem.Bytes()
+
+ clURL := gerritRegex.FindString(string(b))
+ if clURL == "" {
+ return "", errors.New("couldn't get CL URL from gerrit push message")
+ }
+
+ log.Printf("creating gocloud CL... done %s\n", clURL)
+ return clURL, nil
+}
diff --git a/internal/gapicgen/cmd/genbot/genproto.go b/internal/gapicgen/cmd/genbot/genproto.go
new file mode 100644
index 0000000..38330e6
--- /dev/null
+++ b/internal/gapicgen/cmd/genbot/genproto.go
@@ -0,0 +1,168 @@
+// Copyright 2019 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+ "context"
+ "fmt"
+ "log"
+ "os"
+ "os/exec"
+
+ "github.com/google/go-github/github"
+)
+
+const (
+ genprotoBranchName = "regen_genproto"
+ genprotoCommitTitle = "auto-regenerate .pb.go files"
+ genprotoCommitBody = `
+This is an auto-generated regeneration of the .pb.go files by autogogen. Once
+this PR is submitted, autotogen will update the corresponding CL at gocloud
+to depend on the newer version of go-genproto, and assign reviewers. Whilst this
+or any regen PR is open in go-genproto, autogogen will not create any more
+regeneration PRs or CLs. If all regen PRs are closed, autogogen will create a
+new set of regeneration PRs and CLs once per night.
+
+If you have been assigned to review this CL, please:
+
+- Ensure that CI is passin If it's failing, it requires your manual attention.
+- Approve and submit this PR if you believe it's ready to ship. That will prompt
+ autogogen to assign reviewers to the gocloud CL.
+`
+)
+
+// genprotoReviewers is the list of github usernames that will be assigned to
+// review the genproto PR.
+//
+// NOTE: Googler emails will not work - this list must only contain the github
+// username of the reviewer.
+//
+// TODO(ndietz): Can we use github teams?
+var genprotoReviewers = []string{"jadekler", "hongalex", "broady", "noahdietz", "tritone", "codyoss", "tbpg"}
+
+// prGenproto creates a PR for a given genproto change.
+func prGenproto(ctx context.Context, githubClient *github.Client, genprotoDir string) (prNumber int, _ error) {
+ log.Println("creating genproto PR")
+
+ c := exec.Command("/bin/bash", "-c", `
+set -ex
+
+git remote -v
+
+eval $(ssh-agent)
+ssh-add $GITHUB_KEY
+
+git remote add regen_remote $FORK_URL
+git checkout master
+
+git branch -D $BRANCH_NAME || true
+git push -d regen_remote $BRANCH_NAME || true
+
+git add -A
+git checkout -b $BRANCH_NAME
+git commit -m "$COMMIT_TITLE" -m "$COMMIT_BODY"
+git push regen_remote $BRANCH_NAME
+`)
+ c.Stdout = os.Stdout
+ c.Stderr = os.Stderr
+ c.Stdin = os.Stdin // Prevents "the input device is not a TTY" error.
+ c.Env = []string{
+ fmt.Sprintf("PATH=%s", os.Getenv("PATH")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands.
+ fmt.Sprintf("HOME=%s", os.Getenv("HOME")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands.
+ fmt.Sprintf("FORK_URL=%s", fmt.Sprintf("git@github.com:%s/go-genproto.git", *githubUsername)),
+ fmt.Sprintf("COMMIT_TITLE=%s", genprotoCommitTitle),
+ fmt.Sprintf("COMMIT_BODY=%s", genprotoCommitBody),
+ fmt.Sprintf("BRANCH_NAME=%s", genprotoBranchName),
+ fmt.Sprintf("GITHUB_KEY=%s", *githubSSHKeyPath),
+ }
+ c.Dir = genprotoDir
+ if err := c.Run(); err != nil {
+ return 0, err
+ }
+
+ head := fmt.Sprintf("%s:%s", *githubUsername, genprotoBranchName)
+ base := "master"
+ t := genprotoCommitTitle // Because we have to take the address.
+ b := genprotoCommitBody // Because we have to take the address.
+ pr, _, err := githubClient.PullRequests.Create(ctx, "googleapis", "go-genproto", &github.NewPullRequest{
+ Title: &t,
+ Body: &b,
+ Head: &head,
+ Base: &base,
+ })
+ if err != nil {
+ return 0, err
+ }
+
+ // Can't assign the submitter of the PR as a reviewer.
+ var reviewers []string
+ for _, r := range genprotoReviewers {
+ if r != *githubUsername {
+ reviewers = append(reviewers, r)
+ }
+ }
+
+ if _, _, err := githubClient.PullRequests.RequestReviewers(ctx, "googleapis", "go-genproto", pr.GetNumber(), github.ReviewersRequest{
+ Reviewers: reviewers,
+ }); err != nil {
+ return 0, err
+ }
+
+ log.Printf("creating genproto PR... done %s\n", pr.GetHTMLURL())
+
+ return pr.GetNumber(), nil
+}
+
+// amendPRWithCLURL amends the given genproto PR with a link to the given
+// gocloud CL.
+func amendPRWithCLURL(ctx context.Context, githubClient *github.Client, genprotoPRNum int, genprotoDir, gocloudCL string) error {
+ newBody := fmt.Sprintf(`%s
+
+Corresponding gocloud CL: %s
+`, genprotoCommitBody, gocloudCL)
+
+ c := exec.Command("/bin/bash", "-c", `
+set -ex
+
+eval $(ssh-agent)
+ssh-add $GITHUB_KEY
+
+git remote add amend_remote $FORK_URL
+git checkout $BRANCH_NAME
+git commit --amend -m "$COMMIT_TITLE" -m "$COMMIT_BODY"
+git push -f amend_remote $BRANCH_NAME
+`)
+ c.Stdout = os.Stdout
+ c.Stderr = os.Stderr
+ c.Stdin = os.Stdin // Prevents "the input device is not a TTY" error.
+ c.Env = []string{
+ fmt.Sprintf("PATH=%s", os.Getenv("PATH")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands.
+ fmt.Sprintf("HOME=%s", os.Getenv("HOME")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands.
+ fmt.Sprintf("FORK_URL=%s", fmt.Sprintf("git@github.com:%s/go-genproto.git", *githubUsername)),
+ fmt.Sprintf("COMMIT_TITLE=%s", genprotoCommitTitle),
+ fmt.Sprintf("COMMIT_BODY=%s", newBody),
+ fmt.Sprintf("BRANCH_NAME=%s", genprotoBranchName),
+ fmt.Sprintf("GITHUB_KEY=%s", *githubSSHKeyPath),
+ }
+ c.Dir = genprotoDir
+ if err := c.Run(); err != nil {
+ return err
+ }
+
+ _, _, err := githubClient.PullRequests.Edit(ctx, "googleapis", "go-genproto", genprotoPRNum, &github.PullRequest{
+ Body: &newBody,
+ })
+ return err
+}
diff --git a/internal/gapicgen/cmd/genbot/main.go b/internal/gapicgen/cmd/genbot/main.go
new file mode 100644
index 0000000..9e5dc7d
--- /dev/null
+++ b/internal/gapicgen/cmd/genbot/main.go
@@ -0,0 +1,227 @@
+// Copyright 2019 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// genbot is a binary for generating gapics and creating CLs/PRs with the results.
+// It is intended to be used as a bot, though it can be run locally too.
+package main
+
+import (
+ "context"
+ "flag"
+ "fmt"
+ "io/ioutil"
+ "log"
+ "os"
+ "path/filepath"
+
+ "cloud.google.com/go/internal/gapicgen"
+ "cloud.google.com/go/internal/gapicgen/db"
+ "cloud.google.com/go/internal/gapicgen/generator"
+ "github.com/andygrunwald/go-gerrit"
+ "github.com/google/go-github/github"
+ "golang.org/x/oauth2"
+ "golang.org/x/sync/errgroup"
+ "gopkg.in/src-d/go-git.v4"
+)
+
+var (
+ toolsNeeded = []string{"git", "pip", "virtualenv", "python", "go", "protoc", "docker", "artman"}
+
+ accessToken = flag.String("accessToken", "", "Get an access token at https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line")
+ githubUsername = flag.String("githubUsername", "", "ex -githubUsername=jadekler")
+ githubName = flag.String("githubName", "", "ex -githubName=\"Jean de Klerk\"")
+ githubEmail = flag.String("githubEmail", "", "ex -githubEmail=deklerk@google.com")
+ githubSSHKeyPath = flag.String("githubSSHKeyPath", "", "ex -githubSSHKeyPath=/Users/deklerk/.ssh/github_rsa")
+ gerritCookieName = flag.String("gerritCookieName", "", "ex: -gerritCookieName=o")
+ gerritCookieValue = flag.String("gerritCookieValue", "", "ex: -gerritCookieValue=git-your@email.com=SomeHash....")
+
+ usage = func() {
+ fmt.Fprintln(os.Stderr, `genbot \
+ -accessToken=11223344556677889900aabbccddeeff11223344 \
+ -githubUsername=jadekler \
+ -githubEmail=deklerk@google.com \
+ -githubName="Jean de Klerk" \
+ -githubSSHKeyPath=/Users/deklerk/.ssh/github_rsa \
+ -gerritCookieName=o \
+ -gerritCookieValue=git-your@email.com=SomeHash....
+
+-accessToken
+ The access token to authenticate to github. Get this at https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line.
+
+-githubUsername
+ The username to use in the github commit.
+
+-githubName
+ The name to use in the github commit.
+
+-githubEmail
+ The email to use in the github commit.
+
+-githubSSHKeyPath
+ The ssh key to use in the github commit.
+
+-gerritCookieName
+ The name of the cookie. Almost certainly "o".
+
+-gerritCookieValue
+ The value of the gerrit cookie. Probably looks like "git-your@email.com=SomeHash....". Get this at https://code-review.googlesource.com/settings/#HTTPCredentials > Obtain password > "git-your@email.com=SomeHash....".`)
+ os.Exit(2)
+ }
+)
+
+func main() {
+ log.SetFlags(0)
+
+ flag.Usage = usage
+ flag.Parse()
+
+ for k, v := range map[string]string{
+ "accessToken": *accessToken,
+ "githubUsername": *githubUsername,
+ "githubName": *githubName,
+ "githubEmail": *githubEmail,
+ "githubSSHKeyPath": *githubSSHKeyPath,
+ "gerritCookieName": *gerritCookieName,
+ "gerritCookieValue": *gerritCookieValue,
+ } {
+ if v == "" {
+ log.Printf("missing or empty value for required flag --%s\n", k)
+ usage()
+ }
+ }
+
+ ctx := context.Background()
+
+ if err := gapicgen.VerifyAllToolsExist(toolsNeeded); err != nil {
+ log.Fatal(err)
+ }
+
+ // Set up clients.
+
+ if err := gapicgen.SetGitCreds(*githubName, *githubEmail, *gerritCookieName, *gerritCookieValue); err != nil {
+ log.Fatal(err)
+ }
+
+ ts := oauth2.StaticTokenSource(
+ &oauth2.Token{AccessToken: *accessToken},
+ )
+ tc := oauth2.NewClient(ctx, ts)
+ githubClient := github.NewClient(tc)
+
+ gerritClient, err := gerrit.NewClient("https://code-review.googlesource.com", nil)
+ if err != nil {
+ log.Fatal(err)
+ }
+ gerritClient.Authentication.SetCookieAuth(*gerritCookieName, *gerritCookieValue)
+
+ cache := db.New(ctx, githubClient, gerritClient)
+
+ // Get cache.
+
+ prs, err := cache.GetPRs(ctx)
+ if err != nil {
+ log.Fatal(err)
+ }
+
+ cls, err := cache.GetCLs(ctx)
+ if err != nil {
+ log.Fatal(err)
+ }
+
+ // Check if a regen is already underway.
+
+ if pr, ok := db.FirstOpen(prs); ok {
+ log.Printf("there's already a regen underway: %s", pr.URL())
+ return
+ }
+
+ if cl, ok := db.FirstOpen(cls); ok {
+ log.Printf("there's already a regen underway: %s", cl.URL())
+ return
+ }
+
+ // Create temp dirs.
+
+ log.Println("creating temp dir")
+ tmpDir, err := ioutil.TempDir("", "update-genproto")
+ if err != nil {
+ log.Fatal(err)
+ }
+ defer os.RemoveAll(tmpDir)
+
+ log.Printf("working out %s\n", tmpDir)
+
+ googleapisDir := filepath.Join(tmpDir, "googleapis")
+ gocloudDir := filepath.Join(tmpDir, "gocloud")
+ genprotoDir := filepath.Join(tmpDir, "genproto")
+ protoDir := filepath.Join(tmpDir, "proto")
+
+ // Clone repos.
+
+ grp, _ := errgroup.WithContext(ctx)
+ grp.Go(func() error {
+ return gitClone("https://github.com/googleapis/googleapis", googleapisDir)
+ })
+ grp.Go(func() error {
+ return gitClone("https://github.com/googleapis/go-genproto", genprotoDir)
+ })
+ grp.Go(func() error {
+ return gitClone("https://code.googlesource.com/gocloud", gocloudDir)
+ })
+ grp.Go(func() error {
+ return gitClone("https://github.com/google/protobuf", protoDir)
+ })
+ if err := grp.Wait(); err != nil {
+ log.Println(err)
+ }
+
+ // Regen.
+
+ if err := generator.Generate(ctx, googleapisDir, genprotoDir, gocloudDir, protoDir); err != nil {
+ log.Fatal(err)
+ }
+
+ // Create PRs/CLs.
+
+ genprotoPRNum, err := prGenproto(ctx, githubClient, genprotoDir)
+ if err != nil {
+ log.Fatalf("error creating PR for genproto (may need to check logs for more errors): %v", err)
+ }
+
+ gocloudCL, err := clGocloud(ctx, gocloudDir, genprotoPRNum)
+ if err != nil {
+ log.Fatalf("error creating CL for veneers (may need to check logs for more errors): %v", err)
+ }
+
+ if err := amendPRWithCLURL(ctx, githubClient, genprotoPRNum, genprotoDir, gocloudCL); err != nil {
+ log.Fatalf("error amending genproto PR: %v", err)
+ }
+
+ // Log results.
+
+ genprotoPRURL := fmt.Sprintf("https://github.com/googleapis/go-genproto/pull/%d", genprotoPRNum)
+ log.Println(genprotoPRURL)
+ log.Println(gocloudCL)
+}
+
+// gitClone clones a repository in the given directory.
+func gitClone(repo, dir string) error {
+ log.Printf("cloning %s\n", repo)
+
+ _, err := git.PlainClone(dir, false, &git.CloneOptions{
+ URL: repo,
+ Progress: os.Stdout,
+ })
+ return err
+}
diff --git a/internal/gapicgen/cmd/genlocal/README.md b/internal/gapicgen/cmd/genlocal/README.md
new file mode 100644
index 0000000..4890847
--- /dev/null
+++ b/internal/gapicgen/cmd/genlocal/README.md
@@ -0,0 +1,30 @@
+# genlocal
+
+genlocal is a binary for generating gapics locally. It may be used to test out
+new changes, test the generation of a new library, test new generator tweaks,
+run generators against googleapis-private, and various other local tasks.
+
+## Required tools
+
+1. Install [docker](https://www.docker.com/get-started)
+1. Install [protoc](https://github.com/protocolbuffers/protobuf/releases)
+1. Install [Go](http://golang.org/dl)
+1. Install python2.7, pip
+1. Install virtualenv `pip install virtualenv`
+1. Install artman `pip install googleapis-artman`
+1. Install Go tools:
+
+ ```
+ go get \
+ github.com/golang/protobuf/protoc-gen-go \
+ golang.org/x/lint/golint \
+ golang.org/x/tools/cmd/goimports \
+ honnef.co/go/tools/cmd/staticcheck \
+ golang.org/x/review/git-codereview
+ ```
+
+## Running
+
+```
+go run cloud.google.com/go/internal/gapicgen/genlocal
+```
\ No newline at end of file
diff --git a/internal/gapicgen/cmd/genlocal/main.go b/internal/gapicgen/cmd/genlocal/main.go
new file mode 100644
index 0000000..886010a
--- /dev/null
+++ b/internal/gapicgen/cmd/genlocal/main.go
@@ -0,0 +1,112 @@
+// Copyright 2019 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// genlocal is a binary for generating gapics locally. It may be used to test out
+// new changes, test the generation of a new library, test new generator tweaks,
+// run generators against googleapis-private, and various other local tasks.
+package main
+
+import (
+ "context"
+ "flag"
+ "fmt"
+ "io/ioutil"
+ "log"
+ "os"
+ "path/filepath"
+
+ "cloud.google.com/go/internal/gapicgen"
+ "cloud.google.com/go/internal/gapicgen/generator"
+ "golang.org/x/sync/errgroup"
+ "gopkg.in/src-d/go-git.v4"
+)
+
+var (
+ toolsNeeded = []string{"pip", "virtualenv", "python", "go", "protoc", "docker", "artman"}
+
+ usage = func() {
+ fmt.Fprintln(os.Stderr, "genlocal")
+ os.Exit(2)
+ }
+)
+
+func main() {
+ log.SetFlags(0)
+
+ flag.Usage = usage
+ flag.Parse()
+
+ ctx := context.Background()
+
+ if err := gapicgen.VerifyAllToolsExist(toolsNeeded); err != nil {
+ log.Fatal(err)
+ }
+
+ // Create temp dirs.
+
+ log.Println("creating temp dir")
+ tmpDir, err := ioutil.TempDir("", "update-genproto")
+ if err != nil {
+ log.Fatal(err)
+ }
+
+ log.Printf("working out %s\n", tmpDir)
+
+ googleapisDir := filepath.Join(tmpDir, "googleapis")
+ gocloudDir := filepath.Join(tmpDir, "gocloud")
+ genprotoDir := filepath.Join(tmpDir, "genproto")
+ protoDir := filepath.Join(tmpDir, "proto")
+
+ // Clone repos.
+
+ grp, _ := errgroup.WithContext(ctx)
+ grp.Go(func() error {
+ return gitClone("https://github.com/googleapis/googleapis.git", googleapisDir)
+ })
+ grp.Go(func() error {
+ return gitClone("https://github.com/googleapis/go-genproto", genprotoDir)
+ })
+ grp.Go(func() error {
+ return gitClone("https://code.googlesource.com/gocloud", gocloudDir)
+ })
+ grp.Go(func() error {
+ return gitClone("https://github.com/google/protobuf", protoDir)
+ })
+ if err := grp.Wait(); err != nil {
+ log.Println(err)
+ }
+
+ // Regen.
+
+ if err := generator.Generate(ctx, googleapisDir, genprotoDir, gocloudDir, protoDir); err != nil {
+ log.Printf("Generator ran (and failed) in %s\n", tmpDir)
+ log.Fatal(err)
+ }
+
+ // Log results.
+
+ log.Println(genprotoDir)
+ log.Println(gocloudDir)
+}
+
+// gitClone clones a repository in the given directory.
+func gitClone(repo, dir string) error {
+ log.Printf("cloning %s\n", repo)
+
+ _, err := git.PlainClone(dir, false, &git.CloneOptions{
+ URL: repo,
+ Progress: os.Stdout,
+ })
+ return err
+}
diff --git a/internal/gapicgen/cmd/genmgr/Dockerfile b/internal/gapicgen/cmd/genmgr/Dockerfile
new file mode 100644
index 0000000..eb68dc6
--- /dev/null
+++ b/internal/gapicgen/cmd/genmgr/Dockerfile
@@ -0,0 +1,28 @@
+FROM golang:1.13.4-alpine
+
+RUN apk update && \
+ apk add ca-certificates wget git unzip bash
+
+RUN go version
+
+# Install Go tools.
+RUN go get \
+ golang.org/x/lint/golint \
+ golang.org/x/tools/cmd/goimports \
+ honnef.co/go/tools/cmd/staticcheck \
+ golang.org/x/review/git-codereview
+ENV PATH="${PATH}:/root/go/bin"
+
+RUN echo -e '#!/bin/bash\n\
+set -ex\n\
+go run cloud.google.com/go/internal/gapicgen/cmd/genmgr \
+ --accessToken=$ACCESS_TOKEN \
+ --githubName="$GITHUB_NAME" \
+ --githubEmail=$GITHUB_EMAIL \
+ --gerritCookieName=$GERRIT_COOKIE_NAME \
+ --gerritCookieValue=$GERRIT_COOKIE_VALUE \n\
+' >> /run.sh
+RUN chmod a+x /run.sh
+
+WORKDIR /gapicgen
+CMD ["bash", "-c", "/run.sh"]
\ No newline at end of file
diff --git a/internal/gapicgen/cmd/genmgr/README.md b/internal/gapicgen/cmd/genmgr/README.md
new file mode 100644
index 0000000..d57cb2e
--- /dev/null
+++ b/internal/gapicgen/cmd/genmgr/README.md
@@ -0,0 +1,54 @@
+# genmgr
+
+genmgr is a binary used to apply reviewers and update go.mod in a gocloud regen
+CL once the corresponding genproto PR is submitted.
+
+## Required tools
+
+1. Install [Go](http://golang.org/dl)
+1. Install Go tools:
+
+ ```
+ go get \
+ golang.org/x/lint/golint \
+ golang.org/x/tools/cmd/goimports \
+ honnef.co/go/tools/cmd/staticcheck \
+ golang.org/x/review/git-codereview
+ ```
+
+## Getting certs
+
+1. Grab github personal access token (see: https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line)
+2. Grab a gerrit HTTP auth cookie at https://code-review.googlesource.com/settings/#HTTPCredentials > Obtain password > `git-your@email.com=SomeHash....`
+
+## Running locally
+
+Note: this may change your ~/.gitconfig and ~/.gitcookies.
+
+```
+go run cloud.google.com/go/internal/gapicgen/genmgr \
+ --accessToken=11223344556677889900aabbccddeeff11223344 \
+ --githubName="Jean de Klerk" \
+ --githubEmail=deklerk@google.com \
+ --gerritCookieName=o \
+ --gerritCookieValue=<cookie>
+```
+
+## Running with docker
+
+Note: this may leave a lot of docker resources laying around. Use
+`docker system prune` to clean up after runs.
+
+```
+cd /path/to/internal/gapicgen/cmd/genmgr
+docker build . -t genmgr
+docker run -t --rm --privileged \
+ -v `pwd`/../..:/gapicgen \
+ -v /path/to/your/ssh/key/directory:/.ssh \
+ -e "ACCESS_TOKEN=11223344556677889900aabbccddeeff11223344" \
+ -e "GITHUB_NAME=\"Jean de Klerk\"" \
+ -e "GITHUB_EMAIL=deklerk@google.com" \
+ -e "GERRIT_COOKIE_NAME=o" \
+ -e "GERRIT_COOKIE_VALUE=<cookie>" \
+ genmgr
+```
diff --git a/internal/gapicgen/cmd/genmgr/main.go b/internal/gapicgen/cmd/genmgr/main.go
new file mode 100644
index 0000000..5783201
--- /dev/null
+++ b/internal/gapicgen/cmd/genmgr/main.go
@@ -0,0 +1,283 @@
+// Copyright 2019 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// genmgr is a binary used to apply reviewers and update go.mod in a gocloud regen
+// CL once the corresponding genproto PR is submitted.
+package main
+
+import (
+ "context"
+ "flag"
+ "fmt"
+ "io/ioutil"
+ "log"
+ "os"
+ "os/exec"
+ "strings"
+
+ "cloud.google.com/go/internal/gapicgen"
+ "cloud.google.com/go/internal/gapicgen/db"
+ "github.com/andygrunwald/go-gerrit"
+ "github.com/google/go-github/github"
+ "golang.org/x/oauth2"
+)
+
+var (
+ toolsNeeded = []string{"git", "go"}
+
+ gocloudReviewers = []string{"codyoss@google.com", "deklerk@google.com", "tbp@google.com", "cbro@google.com", "hongalex@google.com", "ndietz@google.com", "cjcotter@google.com"}
+
+ accessToken = flag.String("accessToken", "", "Get an access token at https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line")
+ githubName = flag.String("githubName", "", "ex -githubName=\"Jean de Klerk\"")
+ githubEmail = flag.String("githubEmail", "", "ex -githubEmail=deklerk@google.com")
+ gerritCookieName = flag.String("gerritCookieName", "", "ex: -gerritCookieName=o")
+ gerritCookieValue = flag.String("gerritCookieValue", "", "ex: -gerritCookieValue=git-your@email.com=SomeHash....")
+
+ usage = func() {
+ fmt.Fprintln(os.Stderr, `genmgr \
+ -accessToken=11223344556677889900aabbccddeeff11223344 \
+ -gerritCookieName=o \
+ -gerritCookieValue=git-your@email.com=SomeHash....
+
+-accessToken
+ The access token to authenticate to github. Get this at https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line.
+
+-githubName
+ The name to use in the github commit.
+
+-githubEmail
+ The email to use in the github commit.
+
+-gerritCookieName
+ The name of the cookie. Almost certainly "o".
+
+-gerritCookieValue
+ The value of the gerrit cookie. Probably looks like "git-your@email.com=SomeHash....". Get this at https://code-review.googlesource.com/settings/#HTTPCredentials > Obtain password > "git-your@email.com=SomeHash....".`)
+ os.Exit(2)
+ }
+)
+
+func main() {
+ log.SetFlags(0)
+
+ flag.Usage = usage
+ flag.Parse()
+
+ for k, v := range map[string]string{
+ "accessToken": *accessToken,
+ "githubName": *githubName,
+ "githubEmail": *githubEmail,
+ "gerritCookieName": *gerritCookieName,
+ "gerritCookieValue": *gerritCookieValue,
+ } {
+ if v == "" {
+ log.Printf("missing or empty value for required flag --%s\n", k)
+ usage()
+ }
+ }
+
+ ctx := context.Background()
+
+ if err := gapicgen.VerifyAllToolsExist(toolsNeeded); err != nil {
+ log.Fatal(err)
+ }
+
+ // Set up clients.
+
+ if err := gapicgen.SetGitCreds(*githubName, *githubEmail, *gerritCookieName, *gerritCookieValue); err != nil {
+ log.Fatal(err)
+ }
+
+ ts := oauth2.StaticTokenSource(
+ &oauth2.Token{AccessToken: *accessToken},
+ )
+ tc := oauth2.NewClient(ctx, ts)
+ githubClient := github.NewClient(tc)
+
+ gerritClient, err := gerrit.NewClient("https://code-review.googlesource.com", nil)
+ if err != nil {
+ log.Fatal(err)
+ }
+ gerritClient.Authentication.SetCookieAuth(*gerritCookieName, *gerritCookieValue)
+
+ cache := db.New(ctx, githubClient, gerritClient)
+
+ // Get cache.
+
+ prs, err := cache.GetPRs(ctx)
+ if err != nil {
+ log.Fatal(err)
+ }
+
+ cls, err := cache.GetCLs(ctx)
+ if err != nil {
+ log.Fatal(err)
+ }
+
+ // If there's an open PR: no-op! Waiting for someone to submit it.
+ if pr, ok := db.FirstOpen(prs); ok {
+ log.Printf("no work - there's a PR open %s (once it's submitted, we'll have work to do)\n", pr.URL())
+ return
+ }
+
+ // If there's an open CL, let's try working on it.
+ if cl, ok := db.FirstOpen(cls); ok {
+ gerritRA, ok := cl.(*db.GerritRegenAttempt)
+ if !ok {
+ log.Fatalf("got %T, expected GerritRegenAttempt", cl)
+ }
+
+ hasReviewers, err := hasReviewers(gerritClient, gerritRA.ChangeID)
+ if err != nil {
+ log.Fatal(err)
+ }
+
+ // The gerrit cookie encodes username as foo.google.com instead of
+ // foo@google.com. So, if the author is an email, let's strip out
+ // the username part of the email and user that to check for
+ // existence in the cookie.
+ author := cl.Author()
+ if strings.Contains(author, "@") {
+ parts := strings.Split(author, "@")
+ author = parts[0]
+ }
+
+ // If the CL author does not belong to the person running autogogen,
+ // we can't action on it. So: no-op.
+ //
+ // If the CL has reviewers, it must have already had its go.mod
+ // updated. So: no-op.
+ if strings.Contains(*gerritCookieValue, author) && !hasReviewers {
+ log.Printf("it's time to update the regen gocloud CL! (%s)\n", cl.URL())
+
+ if err := finalizeGerritCL(gerritClient, *gerritCookieValue, gerritRA.ChangeID); err != nil {
+ log.Fatal(err)
+ }
+
+ log.Printf("done updating gocloud CL (%s)!\n", cl.URL())
+ } else {
+ log.Printf("there's an open CL (%s) but it doesn't belong to the author running this program\n", cl.URL())
+ }
+ } else {
+ log.Println("there are no open CLs - no work to do!")
+ }
+}
+
+// hasReviewers checks if a given CL has reviewers.
+func hasReviewers(gerritClient *gerrit.Client, changeID string) (bool, error) {
+ ci, _, err := gerritClient.Changes.GetChange(changeID, &gerrit.ChangeOptions{
+ AdditionalFields: []string{
+ "DETAILED_LABELS", // Required to have the Reviewers field populated.
+ "DETAILED_ACCOUNTS", // Required to have Email field populated.
+ },
+ })
+ if err != nil {
+ return false, err
+ }
+
+ // We want to check for any reviewers except kokoro.
+ var reviewersExcludingKokoro []string
+ for _, r := range ci.Reviewers["REVIEWER"] {
+ if strings.Contains(r.Email, "kokoro") {
+ continue
+ }
+ reviewersExcludingKokoro = append(reviewersExcludingKokoro, r.Email)
+ }
+
+ return len(reviewersExcludingKokoro) > 0, nil
+}
+
+// updateAndAddReviewers updates the given CL's go.mod with latest genproto
+// version and adds reviewers.
+func finalizeGerritCL(gerritClient *gerrit.Client, gerritCookieValue, changeID string) error {
+ ci, _, err := gerritClient.Changes.GetChange(changeID, &gerrit.ChangeOptions{
+ AdditionalFields: []string{"CURRENT_REVISION"}, // Required to have the CurrentRevision field populated.
+ })
+ if err != nil {
+ return err
+ }
+
+ cr, ok := ci.Revisions[ci.CurrentRevision]
+ if !ok {
+ return fmt.Errorf("couldn't find current revision %q", ci.CurrentRevision)
+ }
+
+ if err := updateGocloudGoMod(cr.Ref); err != nil {
+ return err
+ }
+
+ return addGocloudReviewers(gerritClient, changeID)
+}
+
+// updateGocloudGoMod updates the go.mod to include latest version of genproto
+// for the given gocloud ref.
+func updateGocloudGoMod(ref string) error {
+ tmpDir, err := ioutil.TempDir("", "finalize-gerrit-cl")
+ if err != nil {
+ return err
+ }
+ defer os.RemoveAll(tmpDir)
+
+ c := exec.Command("/bin/bash", "-c", `
+set -ex
+
+git init
+git remote add origin https://code.googlesource.com/gocloud
+git fetch --all
+git checkout -b finalize_gerrit
+git pull "https://code.googlesource.com/gocloud" $REF
+
+# tidyall
+go mod tidy
+for i in $(find . -name go.mod); do
+ pushd $(dirname $i);
+ # Update genproto and api to latest for every module (latest version is
+ # always correct version). tidy will remove the dependencies if they're not
+ # actually used by the client.
+ go get -u google.golang.org/api | true # We don't care that there's no files at root.
+ go get -u google.golang.org/genproto | true # We don't care that there's no files at root.
+ go mod tidy;
+ popd;
+done
+
+git add -A
+git commit --amend --no-edit
+git-codereview mail
+`)
+ c.Stdout = os.Stdout
+ c.Stderr = os.Stderr
+ c.Stdin = os.Stdin // Prevents "the input device is not a TTY" error.
+ c.Env = []string{
+ fmt.Sprintf("REF=%s", ref),
+ fmt.Sprintf("PATH=%s", os.Getenv("PATH")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands.
+ fmt.Sprintf("HOME=%s", os.Getenv("HOME")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands.
+ }
+ c.Dir = tmpDir
+ return c.Run()
+}
+
+// addGocloudReviewers adds reviewers to the given gocloud CL.
+func addGocloudReviewers(gerritClient *gerrit.Client, changeID string) error {
+ for _, r := range gocloudReviewers {
+ // Can't assign the submitter of the CL as a reviewer.
+ if strings.Contains(*gerritCookieValue, r) {
+ continue
+ }
+ _, _, err := gerritClient.Changes.AddReviewer(changeID, &gerrit.ReviewerInput{Reviewer: r})
+ if err != nil {
+ return err
+ }
+ }
+ return nil
+}
diff --git a/internal/gapicgen/db/db.go b/internal/gapicgen/db/db.go
new file mode 100644
index 0000000..aca1631
--- /dev/null
+++ b/internal/gapicgen/db/db.go
@@ -0,0 +1,192 @@
+// Copyright 2019 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package db is responsible for getting information about CLs and PRs in Gerrit
+// and GitHub respectively.
+package db
+
+import (
+ "context"
+ "fmt"
+ "log"
+ "net/http"
+ "strconv"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/andygrunwald/go-gerrit"
+ "github.com/google/go-github/github"
+)
+
+// RegenAttempt represents either a genproto regen PR or a gocloud gapic regen
+// CL.
+type RegenAttempt interface {
+ Author() string
+ Title() string
+ URL() string
+ Created() time.Time
+ Open() bool
+}
+
+// ByCreated allows RegenAttempt to be sorted by Created field.
+type ByCreated []RegenAttempt
+
+func (a ByCreated) Len() int { return len(a) }
+func (a ByCreated) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+func (a ByCreated) Less(i, j int) bool { return a[i].Created().After(a[j].Created()) }
+
+// regenAttempt represents either a genproto regen PR or a gocloud gapic regen
+// CL.
+type regenAttempt struct {
+ author string
+ title string
+ url string
+ created time.Time
+ open bool
+}
+
+func (ra *regenAttempt) Author() string { return ra.author }
+func (ra *regenAttempt) Title() string { return ra.title }
+func (ra *regenAttempt) URL() string { return ra.url }
+func (ra *regenAttempt) Created() time.Time { return ra.created }
+func (ra *regenAttempt) Open() bool { return ra.open }
+
+// GerritRegenAttempt is a gerrit regen attempt (a CL).
+type GerritRegenAttempt struct {
+ regenAttempt
+ ChangeID string
+}
+
+// GenprotoRegenAttempt is a genproto regen attempt (a PR).
+type GenprotoRegenAttempt struct {
+ regenAttempt
+}
+
+// Db can communicate with GitHub and Gerrit to get PRs / CLs.
+type Db struct {
+ gerritClient *gerrit.Client
+ githubClient *github.Client
+
+ cacheMu sync.Mutex
+ // For some reason, the Changes API only returns AccountID. So we cache the
+ // accountID->name to improve performance / reduce adtl calls.
+ cachedGerritAccounts map[int]string // accountid -> name
+}
+
+// New returns a new Db.
+func New(ctx context.Context, githubClient *github.Client, gerritClient *gerrit.Client) *Db {
+ db := &Db{
+ githubClient: githubClient,
+ gerritClient: gerritClient,
+
+ cacheMu: sync.Mutex{},
+ cachedGerritAccounts: map[int]string{},
+ }
+
+ return db
+}
+
+// GetPRs fetches regen PRs from genproto.
+func (db *Db) GetPRs(ctx context.Context) ([]RegenAttempt, error) {
+ log.Println("getting genproto changes")
+ genprotoPRs := []RegenAttempt{}
+
+ // We don't bother paginating, because it hurts our requests quota and makes
+ // the page slower without a lot of value.
+ opt := &github.PullRequestListOptions{
+ ListOptions: github.ListOptions{PerPage: 50},
+ State: "all",
+ }
+ prs, _, err := db.githubClient.PullRequests.List(ctx, "googleapis", "go-genproto", opt)
+ if err != nil {
+ return nil, err
+ }
+ for _, pr := range prs {
+ if !strings.Contains(pr.GetTitle(), "regen") {
+ continue
+ }
+ genprotoPRs = append(genprotoPRs, &GenprotoRegenAttempt{
+ regenAttempt: regenAttempt{
+ author: pr.GetUser().GetLogin(),
+ title: pr.GetTitle(),
+ url: pr.GetHTMLURL(),
+ created: pr.GetCreatedAt(),
+ open: pr.GetState() == "open",
+ },
+ })
+ }
+ return genprotoPRs, nil
+}
+
+// GetCLs fetches regen CLs from Gerrit.
+func (db *Db) GetCLs(ctx context.Context) ([]RegenAttempt, error) {
+ log.Println("getting gocloud changes")
+ gocloudCLs := []RegenAttempt{}
+
+ changes, _, err := db.gerritClient.Changes.QueryChanges(&gerrit.QueryChangeOptions{
+ QueryOptions: gerrit.QueryOptions{Query: []string{"project:gocloud"}, Limit: 200},
+ })
+ if err != nil {
+ return nil, err
+ }
+
+ for _, c := range *changes {
+ if !strings.Contains(c.Subject, "regen") {
+ continue
+ }
+
+ // For some reason, the Changes API only returns AccountID. So now we
+ // have to go get the name.
+ db.cacheMu.Lock()
+ if _, ok := db.cachedGerritAccounts[c.Owner.AccountID]; !ok {
+ log.Println("looking up user", c.Owner.AccountID)
+ ai, resp, err := db.gerritClient.Accounts.GetAccount(strconv.Itoa(c.Owner.AccountID))
+ if err != nil {
+ if resp.StatusCode == http.StatusNotFound {
+ db.cachedGerritAccounts[c.Owner.AccountID] = fmt.Sprintf("unknown user account ID: %d\n", c.Owner.AccountID)
+ } else {
+ db.cacheMu.Unlock()
+ return nil, err
+ }
+ } else {
+ db.cachedGerritAccounts[c.Owner.AccountID] = ai.Email
+ }
+ }
+
+ gocloudCLs = append(gocloudCLs, &GerritRegenAttempt{
+ regenAttempt: regenAttempt{
+ author: db.cachedGerritAccounts[c.Owner.AccountID],
+ title: c.Subject,
+ url: fmt.Sprintf("https://code-review.googlesource.com/q/%s", c.ChangeID),
+ created: c.Created.Time,
+ open: c.Status == "NEW",
+ },
+ ChangeID: c.ChangeID,
+ })
+ db.cacheMu.Unlock()
+ }
+
+ return gocloudCLs, nil
+}
+
+// FirstOpen returns the first open regen attempt.
+func FirstOpen(ras []RegenAttempt) (RegenAttempt, bool) {
+ for _, ra := range ras {
+ if ra.Open() {
+ return ra, true
+ }
+ }
+ return nil, false
+}
diff --git a/internal/gapicgen/generator/config.go b/internal/gapicgen/generator/config.go
new file mode 100644
index 0000000..a09b98d
--- /dev/null
+++ b/internal/gapicgen/generator/config.go
@@ -0,0 +1,250 @@
+// Copyright 2019 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package generator
+
+// microgenConfig represents a single microgen target.
+type microgenConfig struct {
+ // inputDirectoryPath is the path to the input (.proto, etc) files, relative
+ // to googleapisDir.
+ inputDirectoryPath string
+
+ // importPath is the path that this library should be imported as.
+ importPath string
+
+ // pkg is the name that should be used in the package declaration.
+ pkg string
+
+ // gRPCServiceConfigPath is the path to the grpc service config for this
+ // target, relative to googleapisDir.
+ gRPCServiceConfigPath string
+
+ // apiServiceConfigPath is the path to the gapic service config for this
+ // target, relative to googleapisDir.
+ apiServiceConfigPath string
+
+ // releaseLevel is the release level of this target. Values incl ga,
+ // beta, alpha.
+ releaseLevel string
+}
+
+var microgenGapicConfigs = []*microgenConfig{
+ {
+ inputDirectoryPath: "google/cloud/texttospeech/v1",
+ pkg: "texttospeech",
+ importPath: "cloud.google.com/go/texttospeech/apiv1",
+ gRPCServiceConfigPath: "google/cloud/texttospeech/v1/texttospeech_grpc_service_config.json",
+ apiServiceConfigPath: "google/cloud/texttospeech/v1/texttospeech_v1.yaml",
+ releaseLevel: "alpha",
+ },
+ {
+ inputDirectoryPath: "google/cloud/asset/v1",
+ pkg: "asset",
+ importPath: "cloud.google.com/go/asset/apiv1",
+ gRPCServiceConfigPath: "google/cloud/asset/v1/cloudasset_grpc_service_config.json",
+ apiServiceConfigPath: "google/cloud/asset/v1/cloudasset_v1.yaml",
+ releaseLevel: "alpha",
+ },
+ {
+ inputDirectoryPath: "google/cloud/language/v1",
+ pkg: "language",
+ importPath: "cloud.google.com/go/language/apiv1",
+ gRPCServiceConfigPath: "google/cloud/language/v1/language_grpc_service_config.json",
+ apiServiceConfigPath: "google/cloud/language/language_v1.yaml",
+ releaseLevel: "alpha",
+ },
+ {
+ inputDirectoryPath: "google/cloud/phishingprotection/v1beta1",
+ pkg: "phishingprotection",
+ importPath: "cloud.google.com/go/phishingprotection/apiv1beta1",
+ gRPCServiceConfigPath: "google/cloud/phishingprotection/v1beta1/phishingprotection_grpc_service_config.json",
+ apiServiceConfigPath: "google/cloud/phishingprotection/v1beta1/phishingprotection_v1beta1.yaml",
+ releaseLevel: "beta",
+ },
+ {
+ inputDirectoryPath: "google/cloud/translate/v3",
+ pkg: "translate",
+ importPath: "cloud.google.com/go/translate/apiv3",
+ gRPCServiceConfigPath: "google/cloud/translate/v3/translate_grpc_service_config.json",
+ apiServiceConfigPath: "google/cloud/translate/v3/translate_v3.yaml",
+ releaseLevel: "ga",
+ },
+ {
+ inputDirectoryPath: "google/cloud/scheduler/v1",
+ pkg: "scheduler",
+ importPath: "cloud.google.com/go/scheduler/apiv1",
+ gRPCServiceConfigPath: "google/cloud/scheduler/v1/cloudscheduler_grpc_service_config.json",
+ apiServiceConfigPath: "google/cloud/scheduler/v1/cloudscheduler_v1.yaml",
+ releaseLevel: "ga",
+ },
+ {
+ inputDirectoryPath: "google/cloud/scheduler/v1beta1",
+ pkg: "scheduler",
+ importPath: "cloud.google.com/go/scheduler/apiv1beta1",
+ gRPCServiceConfigPath: "google/cloud/scheduler/v1beta1/cloudscheduler_grpc_service_config.json",
+ apiServiceConfigPath: "google/cloud/scheduler/v1beta1/cloudscheduler_v1beta1.yaml",
+ releaseLevel: "beta",
+ },
+ {
+ inputDirectoryPath: "google/cloud/speech/v1",
+ pkg: "speech",
+ importPath: "cloud.google.com/go/speech/apiv1",
+ gRPCServiceConfigPath: "google/cloud/speech/v1/speech_grpc_service_config.json",
+ apiServiceConfigPath: "google/cloud/speech/v1/speech_v1.yaml",
+ releaseLevel: "ga",
+ },
+ {
+ inputDirectoryPath: "google/cloud/speech/v1p1beta1",
+ pkg: "speech",
+ importPath: "cloud.google.com/go/speech/apiv1p1beta1",
+ gRPCServiceConfigPath: "google/cloud/speech/v1p1beta1/speech_grpc_service_config.json",
+ apiServiceConfigPath: "google/cloud/speech/v1p1beta1/speech_v1p1beta1.yaml",
+ releaseLevel: "beta",
+ },
+ {
+ inputDirectoryPath: "google/cloud/bigquery/datatransfer/v1",
+ pkg: "datatransfer",
+ importPath: "cloud.google.com/go/bigquery/datatransfer/apiv1",
+ gRPCServiceConfigPath: "google/cloud/bigquery/datatransfer/v1/bigquerydatatransfer_grpc_service_config.json",
+ apiServiceConfigPath: "google/cloud/bigquery/datatransfer/v1/bigquerydatatransfer_v1.yaml",
+ releaseLevel: "alpha",
+ },
+ {
+ inputDirectoryPath: "google/cloud/bigquery/storage/v1beta1",
+ pkg: "storage",
+ importPath: "cloud.google.com/go/bigquery/storage/apiv1beta1",
+ gRPCServiceConfigPath: "google/cloud/bigquery/storage/v1beta1/bigquerystorage_grpc_service_config.json",
+ apiServiceConfigPath: "google/cloud/bigquery/storage/v1beta1/bigquerystorage_v1beta1.yaml",
+ releaseLevel: "beta",
+ },
+ {
+ inputDirectoryPath: "google/cloud/iot/v1",
+ pkg: "iot",
+ importPath: "cloud.google.com/go/iot/apiv1",
+ gRPCServiceConfigPath: "google/cloud/iot/v1/cloudiot_grpc_service_config.json",
+ apiServiceConfigPath: "google/cloud/iot/v1/cloudiot_v1.yaml",
+ releaseLevel: "ga",
+ },
+ {
+ inputDirectoryPath: "google/cloud/recommender/v1beta1",
+ pkg: "recommender",
+ importPath: "cloud.google.com/go/recommender/apiv1beta1",
+ gRPCServiceConfigPath: "google/cloud/recommender/v1beta1/recommender_grpc_service_config.json",
+ apiServiceConfigPath: "google/cloud/recommender/v1beta1/recommender_v1beta1.yaml",
+ releaseLevel: "beta",
+ },
+ {
+ inputDirectoryPath: "google/cloud/tasks/v2",
+ pkg: "cloudtasks",
+ importPath: "cloud.google.com/go/cloudtasks/apiv2",
+ gRPCServiceConfigPath: "google/cloud/tasks/v2/cloudtasks_grpc_service_config.json",
+ apiServiceConfigPath: "google/cloud/tasks/v2/cloudtasks_v2.yaml",
+ releaseLevel: "ga",
+ },
+ {
+ inputDirectoryPath: "google/cloud/tasks/v2beta2",
+ pkg: "cloudtasks",
+ importPath: "cloud.google.com/go/cloudtasks/apiv2beta2",
+ gRPCServiceConfigPath: "google/cloud/tasks/v2beta2/cloudtasks_grpc_service_config.json",
+ apiServiceConfigPath: "google/cloud/tasks/v2beta2/cloudtasks_v2beta2.yaml",
+ releaseLevel: "beta",
+ },
+ {
+ inputDirectoryPath: "google/cloud/tasks/v2beta3",
+ pkg: "cloudtasks",
+ importPath: "cloud.google.com/go/cloudtasks/apiv2beta3",
+ gRPCServiceConfigPath: "google/cloud/tasks/v2beta3/cloudtasks_grpc_service_config.json",
+ apiServiceConfigPath: "google/cloud/tasks/v2beta3/cloudtasks_v2beta3.yaml",
+ releaseLevel: "beta",
+ },
+ {
+ inputDirectoryPath: "google/cloud/videointelligence/v1",
+ pkg: "videointelligence",
+ importPath: "cloud.google.com/go/videointelligence/apiv1",
+ gRPCServiceConfigPath: "google/cloud/videointelligence/v1/videointelligence_grpc_service_config.json",
+ apiServiceConfigPath: "google/cloud/videointelligence/v1/videointelligence_v1.yaml",
+ releaseLevel: "alpha",
+ },
+ // TODO(ndietz): Microgen for this gapic is currently broken. Investigating.
+ // {
+ // inputDirectoryPath: "google/cloud/vision/v1",
+ // pkg: "vision",
+ // importPath: "cloud.google.com/go/vision/apiv1",
+ // gRPCServiceConfigPath: "google/cloud/vision/v1/vision_grpc_service_config.json",
+ // apiServiceConfigPath: "google/cloud/vision/v1/vision_v1.yaml",
+ // releaseLevel: "ga",
+ // },
+ {
+ inputDirectoryPath: "google/cloud/webrisk/v1beta1",
+ pkg: "webrisk",
+ importPath: "cloud.google.com/go/webrisk/apiv1beta1",
+ gRPCServiceConfigPath: "google/cloud/webrisk/v1beta1/webrisk_grpc_service_config.json",
+ apiServiceConfigPath: "google/cloud/webrisk/v1beta1/webrisk_v1beta1.yaml",
+ releaseLevel: "beta",
+ },
+}
+
+// Relative to gocloud dir.
+var gapicsWithManual = []string{
+ "errorreporting/apiv1beta1",
+ "firestore/apiv1beta1",
+ "firestore/apiv1",
+ "logging/apiv2",
+ "longrunning/autogen",
+ "pubsub/apiv1",
+ "spanner/apiv1",
+ "trace/apiv1",
+}
+
+// Relative to googleapis dir.
+var artmanGapicConfigPaths = []string{
+ "google/api/expr/artman_cel.yaml",
+ "google/cloud/asset/artman_cloudasset_v1beta1.yaml",
+ "google/cloud/asset/artman_cloudasset_v1p2beta1.yaml",
+ "google/iam/credentials/artman_iamcredentials_v1.yaml",
+ "google/cloud/automl/artman_automl_v1.yaml",
+ "google/cloud/automl/artman_automl_v1beta1.yaml",
+ "google/cloud/dataproc/artman_dataproc_v1.yaml",
+ "google/cloud/dataproc/artman_dataproc_v1beta2.yaml",
+ "google/cloud/dialogflow/v2/artman_dialogflow_v2.yaml",
+ "google/cloud/irm/artman_irm_v1alpha2.yaml",
+ "google/cloud/kms/artman_cloudkms.yaml",
+ "google/cloud/language/artman_language_v1beta2.yaml",
+ "google/cloud/oslogin/artman_oslogin_v1.yaml",
+ "google/cloud/oslogin/artman_oslogin_v1beta.yaml",
+ "google/cloud/recaptchaenterprise/artman_recaptchaenterprise_v1beta1.yaml",
+ "google/cloud/redis/artman_redis_v1beta1.yaml",
+ "google/cloud/redis/artman_redis_v1.yaml",
+ "google/cloud/securitycenter/artman_securitycenter_v1beta1.yaml",
+ "google/cloud/securitycenter/artman_securitycenter_v1.yaml",
+ "google/cloud/talent/artman_talent_v4beta1.yaml",
+ "google/cloud/videointelligence/artman_videointelligence_v1beta2.yaml",
+ "google/cloud/vision/artman_vision_v1p1beta1.yaml",
+ "google/devtools/artman_clouddebugger.yaml",
+ "google/devtools/cloudbuild/artman_cloudbuild.yaml",
+ "google/devtools/clouderrorreporting/artman_errorreporting.yaml",
+ "google/devtools/cloudtrace/artman_cloudtrace_v1.yaml",
+ "google/devtools/cloudtrace/artman_cloudtrace_v2.yaml",
+ "google/devtools/containeranalysis/artman_containeranalysis_v1beta1.yaml",
+ "google/firestore/artman_firestore.yaml",
+ "google/firestore/admin/artman_firestore_v1.yaml",
+ "google/logging/artman_logging.yaml",
+ "google/longrunning/artman_longrunning.yaml",
+ "google/monitoring/artman_monitoring.yaml",
+ "google/privacy/dlp/artman_dlp_v2.yaml",
+ "google/pubsub/artman_pubsub.yaml",
+ "google/spanner/admin/database/artman_spanner_admin_database.yaml",
+ "google/spanner/admin/instance/artman_spanner_admin_instance.yaml",
+ "google/spanner/artman_spanner.yaml",
+}
diff --git a/internal/gapicgen/generator/gapics.go b/internal/gapicgen/generator/gapics.go
new file mode 100644
index 0000000..9477ce2
--- /dev/null
+++ b/internal/gapicgen/generator/gapics.go
@@ -0,0 +1,323 @@
+// Copyright 2019 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package generator
+
+import (
+ "bytes"
+ "context"
+ "fmt"
+ "io"
+ "log"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "regexp"
+ "strings"
+)
+
+var dockerPullRegex = regexp.MustCompile("(googleapis/artman:[0-9]+.[0-9]+.[0-9]+)")
+
+// generateGapics generates gapics.
+func generateGapics(ctx context.Context, googleapisDir, protoDir, gocloudDir, genprotoDir string) error {
+ for _, c := range artmanGapicConfigPaths {
+ if err := artman(c, googleapisDir); err != nil {
+ return err
+ }
+ }
+
+ if err := copyArtmanFiles(googleapisDir, gocloudDir); err != nil {
+ return err
+ }
+
+ for _, c := range microgenGapicConfigs {
+ if err := microgen(c, googleapisDir, protoDir, gocloudDir); err != nil {
+ return err
+ }
+ }
+
+ if err := copyMicrogenFiles(gocloudDir); err != nil {
+ return err
+ }
+
+ if err := setVersion(gocloudDir); err != nil {
+ return err
+ }
+
+ for _, m := range gapicsWithManual {
+ if err := setGoogleClientInfo(gocloudDir + "/" + m); err != nil {
+ return err
+ }
+ }
+
+ if err := addModReplaceGenproto(gocloudDir, genprotoDir); err != nil {
+ return err
+ }
+
+ if err := vet(gocloudDir); err != nil {
+ return err
+ }
+
+ if err := build(gocloudDir); err != nil {
+ return err
+ }
+
+ if err := dropModReplaceGenproto(gocloudDir); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+// installMicrogen installs the microgen gapic protoc plugin.
+//
+// The installation could happen during the docker build phase, but it would
+// mean needing to rebuild the docker image each time a new version of the
+// plugin is available.
+//
+// The plugin version could be specified in genbot's go.mod, but since there's
+// a mandatory copy phase it seems reasonable to just do it all inline.
+func installMicrogen() error {
+ c := exec.Command("go", "get", "-u", "github.com/golang/protobuf/protoc-gen-go")
+ c.Stdout = os.Stdout
+ c.Stderr = os.Stderr
+ c.Env = []string{
+ fmt.Sprintf("GOPATH=%s", os.Getenv("GOPATH")),
+ fmt.Sprintf("PATH=%s", os.Getenv("PATH")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands.
+ fmt.Sprintf("HOME=%s", os.Getenv("HOME")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands.
+ }
+ if err := c.Run(); err != nil {
+ return err
+ }
+
+ c = exec.Command("cp", "$GOPATH/protoc-gen-go", "/export/bin/protoc-gen-go")
+ c.Stdout = os.Stdout
+ c.Stderr = os.Stderr
+ c.Env = []string{
+ fmt.Sprintf("GOPATH=%s", os.Getenv("GOPATH")),
+ }
+ return c.Run()
+}
+
+// addModReplaceGenproto adds a genproto replace statement that points genproto
+// to the local copy. This is necessary since the remote genproto may not have
+// changes that are necessary for the in-flight regen.
+func addModReplaceGenproto(gocloudDir, genprotoDir string) error {
+ c := exec.Command("bash", "-c", `
+set -ex
+
+GENPROTO_VERSION=$(cat go.mod | cat go.mod | grep genproto | awk '{print $2}')
+go mod edit -replace "google.golang.org/genproto@$GENPROTO_VERSION=$GENPROTO_DIR"
+`)
+ c.Stdout = os.Stdout
+ c.Stderr = os.Stderr
+ c.Dir = gocloudDir
+ c.Env = []string{
+ "GENPROTO_DIR=" + genprotoDir,
+ fmt.Sprintf("PATH=%s", os.Getenv("PATH")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands.
+ fmt.Sprintf("HOME=%s", os.Getenv("HOME")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands.
+ }
+ return c.Run()
+}
+
+// dropModReplaceGenproto drops the genproto replace statement. It is intended
+// to be run after addModReplaceGenproto.
+func dropModReplaceGenproto(gocloudDir string) error {
+ c := exec.Command("bash", "-c", `
+set -ex
+
+GENPROTO_VERSION=$(cat go.mod | cat go.mod | grep genproto | grep -v replace | awk '{print $2}')
+go mod edit -dropreplace "google.golang.org/genproto@$GENPROTO_VERSION"
+`)
+ c.Stdout = os.Stdout
+ c.Stderr = os.Stderr
+ c.Dir = gocloudDir
+ c.Env = []string{
+ fmt.Sprintf("PATH=%s", os.Getenv("PATH")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands.
+ fmt.Sprintf("HOME=%s", os.Getenv("HOME")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands.
+ }
+ return c.Run()
+}
+
+// setGoogleClientInfo enters a directory and updates setGoogleClientInfo
+// to be public. It is used for gapics which have manuals that use them, since
+// the manual needs to call this function.
+func setGoogleClientInfo(manualDir string) error {
+ // TODO(deklerk): Migrate this all to Go instead of using bash.
+
+ c := exec.Command("bash", "-c", `
+find . -name '*.go' -exec sed -i.backup -e 's/setGoogleClientInfo/SetGoogleClientInfo/g' '{}' '+'
+find . -name '*.backup' -delete
+`)
+ c.Stdout = os.Stdout
+ c.Stderr = os.Stderr
+ c.Dir = manualDir
+ return c.Run()
+}
+
+// setVersion updates the versionClient constant in all .go files. It may create
+// .backup files on certain systems (darwin), and so should be followed by a
+// clean-up of .backup files.
+func setVersion(gocloudDir string) error {
+ // TODO(deklerk): Migrate this all to Go instead of using bash.
+
+ c := exec.Command("bash", "-c", `
+ver=$(date +%Y%m%d)
+git ls-files -mo | while read modified; do
+ dir=${modified%/*.*}
+ find . -path "*/$dir/doc.go" -exec sed -i.backup -e "s/^const versionClient.*/const versionClient = \"$ver\"/" '{}' +;
+done
+find . -name '*.backup' -delete
+`)
+ c.Stdout = os.Stdout
+ c.Stderr = os.Stderr
+ c.Dir = gocloudDir
+ return c.Run()
+}
+
+// artman runs artman on a single artman gapic config path.
+func artman(gapicConfigPath, googleapisDir string) error {
+ log.Println("artman generating", gapicConfigPath)
+
+ // Write command output to both os.Stderr and local, so that we can check
+ // for `Cannot find artman Docker image. Run `docker pull googleapis/artman:0.41.0` to pull the image.`.
+ inmem := bytes.NewBuffer([]byte{})
+ w := io.MultiWriter(os.Stderr, inmem)
+
+ c := exec.Command("artman", "--config", gapicConfigPath, "generate", "go_gapic")
+ c.Stdout = os.Stdout
+ c.Stderr = w
+ c.Stdin = os.Stdin // Prevents "the input device is not a TTY" error.
+ c.Dir = googleapisDir
+ if err := c.Run(); err == nil {
+ return nil
+ } else {
+ stderr := inmem.Bytes()
+ if dockerPullRegex.Match(stderr) {
+ artmanImg := dockerPullRegex.FindString(string(stderr))
+ c := exec.Command("docker", "pull", artmanImg)
+ c.Stdout = os.Stdout
+ c.Stderr = os.Stderr
+ c.Stdin = os.Stdin // Prevents "the input device is not a TTY" error.
+ if err := c.Run(); err != nil {
+ return err
+ }
+ } else {
+ return err
+ }
+ }
+
+ // If the last command failed, and we were able to fix it with `docker pull`,
+ // then let's try regenerating. When https://github.com/googleapis/artman/issues/732
+ // is solved, we won't have to do this.
+ c = exec.Command("artman", "--config", gapicConfigPath, "generate", "go_gapic")
+ c.Stdout = os.Stdout
+ c.Stderr = os.Stderr
+ c.Stdin = os.Stdin // Prevents "the input device is not a TTY" error.
+ c.Dir = googleapisDir
+ return c.Run()
+}
+
+// microgen runs the microgenerator on a single microgen config.
+func microgen(conf *microgenConfig, googleapisDir, protoDir, gocloudDir string) error {
+ log.Println("microgen generating", conf.pkg)
+
+ var protoFiles []string
+ if err := filepath.Walk(googleapisDir+"/"+conf.inputDirectoryPath, func(path string, info os.FileInfo, err error) error {
+ if err != nil {
+ return err
+ }
+ if strings.Contains(info.Name(), ".proto") {
+ protoFiles = append(protoFiles, path)
+ }
+ return nil
+ }); err != nil {
+ return err
+ }
+
+ args := []string{"-I", googleapisDir,
+ "-I", protoDir,
+ "--go_gapic_out", gocloudDir,
+ "--go_gapic_opt", fmt.Sprintf("go-gapic-package=%s;%s", conf.importPath, conf.pkg),
+ "--go_gapic_opt", fmt.Sprintf("grpc-service-config=%s", conf.gRPCServiceConfigPath),
+ "--go_gapic_opt", fmt.Sprintf("gapic-service-config=%s", conf.apiServiceConfigPath),
+ "--go_gapic_opt", fmt.Sprintf("release-level=%s", conf.releaseLevel)}
+ args = append(args, protoFiles...)
+ c := exec.Command("protoc", args...)
+ c.Stdout = os.Stdout
+ c.Stderr = os.Stderr
+ c.Dir = googleapisDir
+ return c.Run()
+}
+
+// copyMicrogenFiles takes microgen files from gocloudDir/cloud.google.com/go
+// and places them in gocloudDir.
+func copyMicrogenFiles(gocloudDir string) error {
+ // The period at the end is analagous to * (copy everything in this dir).
+ c := exec.Command("cp", "-R", gocloudDir+"/cloud.google.com/go/.", ".")
+ c.Stdout = os.Stdout
+ c.Stderr = os.Stderr
+ c.Dir = gocloudDir
+ if err := c.Run(); err != nil {
+ return err
+ }
+
+ c = exec.Command("rm", "-rf", "cloud.google.com")
+ c.Stdout = os.Stdout
+ c.Stderr = os.Stderr
+ c.Dir = gocloudDir
+ return c.Run()
+}
+
+// gapiFolderRegex finds gapi folders, such as gapi-cloud-cel-go/cloud.google.com/go
+// in paths like [...]/artman-genfiles/gapi-cloud-cel-go/cloud.google.com/go/expr/apiv1alpha1/cel_client.go.
+var gapiFolderRegex = regexp.MustCompile("gapi-.+/cloud.google.com/go/")
+
+// copyArtmanFiles copies artman files from the generated googleapisDir location
+// to their appropriate spots in gocloudDir.
+func copyArtmanFiles(googleapisDir, gocloudDir string) error {
+ // For some reason os.Exec doesn't like to cp globs, so we can't do the
+ // much simpler cp -r <googleapisDir>/artman-genfiles/gapi-*/cloud.google.com/go/* <gocloudDir>.
+ //
+ // (Possibly only specific to /var/folders (os.Tmpdir()) on darwin?)
+ gapiFolders := make(map[string]struct{})
+ root := googleapisDir + "/artman-genfiles"
+ if err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
+ if err != nil {
+ return err
+ }
+ // Things like [...]/artman-genfiles/gapi-cloud-cel-go/cloud.google.com/go/expr/apiv1alpha1/cel_client.go
+ // become gapi-cloud-cel-go/cloud.google.com/go/.
+ //
+ // The period at the end is analagous to * (copy everything in this dir).
+ if gapiFolderRegex.MatchString(path) {
+ gapiFolders[root+"/"+gapiFolderRegex.FindString(path)+"."] = struct{}{}
+ }
+ return nil
+ }); err != nil {
+ return err
+ }
+
+ for f := range gapiFolders {
+ c := exec.Command("cp", "-R", f, gocloudDir)
+ c.Stdout = os.Stdout
+ c.Stderr = os.Stderr
+ c.Dir = googleapisDir
+ if err := c.Run(); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
diff --git a/internal/gapicgen/generator/generator.go b/internal/gapicgen/generator/generator.go
new file mode 100644
index 0000000..0841413
--- /dev/null
+++ b/internal/gapicgen/generator/generator.go
@@ -0,0 +1,62 @@
+// Copyright 2019 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package generator provides tools for generating clients.
+package generator
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "os/exec"
+)
+
+// Generate generates genproto and gapics.
+func Generate(ctx context.Context, googleapisDir, genprotoDir, gocloudDir, protoDir string) error {
+ if err := regenGenproto(ctx, genprotoDir, googleapisDir, protoDir); err != nil {
+ return fmt.Errorf("error generating genproto (may need to check logs for more errors): %v", err)
+ }
+
+ if err := generateGapics(ctx, googleapisDir, protoDir, gocloudDir, genprotoDir); err != nil {
+ return fmt.Errorf("error generating gapics (may need to check logs for more errors): %v", err)
+ }
+
+ return nil
+}
+
+// build attemps to build all packages recursively from the given directory.
+func build(dir string) error {
+ c := exec.Command("go", "build", "./...")
+ c.Stdout = os.Stdout
+ c.Stderr = os.Stderr
+ c.Dir = dir
+ return c.Run()
+}
+
+// vet runs linters on all .go files recursively from the given directory.
+func vet(dir string) error {
+ c := exec.Command("goimports", "-w", ".")
+ c.Stdout = os.Stdout
+ c.Stderr = os.Stderr
+ c.Dir = dir
+ if err := c.Run(); err != nil {
+ return err
+ }
+
+ c = exec.Command("gofmt", "-s", "-d", "-w", "-l", ".")
+ c.Stdout = os.Stdout
+ c.Stderr = os.Stderr
+ c.Dir = dir
+ return c.Run()
+}
diff --git a/internal/gapicgen/generator/genproto.go b/internal/gapicgen/generator/genproto.go
new file mode 100644
index 0000000..9bb9a06
--- /dev/null
+++ b/internal/gapicgen/generator/genproto.go
@@ -0,0 +1,200 @@
+// Copyright 2019 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package generator
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io/ioutil"
+ "log"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "regexp"
+ "strconv"
+ "strings"
+
+ "golang.org/x/sync/errgroup"
+)
+
+var goPkgOptRe = regexp.MustCompile(`(?m)^option go_package = (.*);`)
+
+// regenGenproto regenerates the genproto repository.
+//
+// regenGenproto recursively walks through each directory named by given
+// arguments, looking for all .proto files. (Symlinks are not followed.) Any
+// proto file without `go_package` option or whose option does not begin with
+// the genproto prefix is ignored.
+//
+// If multiple roots contain files with the same name, eg "root1/path/to/file"
+// and "root2/path/to/file", only the first file is processed; the rest are
+// ignored.
+//
+// Protoc is executed on remaining files, one invocation per set of files
+// declaring the same Go package.
+func regenGenproto(ctx context.Context, genprotoDir, googleapisDir, protoDir string) error {
+ log.Println("regenerating genproto")
+
+ // The protoc include directory is actually the "src" directory of the repo.
+ protoDir += "/src"
+
+ // Create space to put generated .pb.go's.
+ c := exec.Command("mkdir", "generated")
+ c.Stdout = os.Stdout
+ c.Stderr = os.Stderr
+ c.Dir = genprotoDir
+ if err := c.Run(); err != nil {
+ return err
+ }
+
+ // Record and map all .proto files to their Go packages.
+ seenFiles := make(map[string]bool)
+ pkgFiles := make(map[string][]string)
+ for _, root := range []string{googleapisDir, protoDir} {
+ walkFn := func(path string, info os.FileInfo, err error) error {
+ if err != nil {
+ return err
+ }
+ if !info.Mode().IsRegular() || !strings.HasSuffix(path, ".proto") {
+ return nil
+ }
+
+ switch rel, err := filepath.Rel(root, path); {
+ case err != nil:
+ return err
+ case seenFiles[rel]:
+ return nil
+ default:
+ seenFiles[rel] = true
+ }
+
+ pkg, err := goPkg(path)
+ if err != nil {
+ return err
+ }
+ pkgFiles[pkg] = append(pkgFiles[pkg], path)
+ return nil
+ }
+ if err := filepath.Walk(root, walkFn); err != nil {
+ return err
+ }
+ }
+
+ if len(pkgFiles) == 0 {
+ return errors.New("couldn't find any pkgfiles")
+ }
+
+ // Run protoc on all protos of all packages.
+ grp, _ := errgroup.WithContext(ctx)
+ for pkg, fnames := range pkgFiles {
+ if !strings.HasPrefix(pkg, "google.golang.org/genproto") {
+ continue
+ }
+ pk := pkg
+ fn := fnames
+ grp.Go(func() error {
+ log.Println("running protoc on", pk)
+ return protoc(genprotoDir, googleapisDir, protoDir, fn)
+ })
+ }
+ if err := grp.Wait(); err != nil {
+ return err
+ }
+
+ // Move all generated content to their correct locations in the repository,
+ // because protoc puts it in a folder called generated/.
+
+ // The period at the end is analagous to * (copy everything in this dir).
+ c = exec.Command("cp", "-R", "generated/google.golang.org/genproto/.", ".")
+ c.Stdout = os.Stdout
+ c.Stderr = os.Stderr
+ c.Dir = genprotoDir
+ if err := c.Run(); err != nil {
+ return err
+ }
+
+ c = exec.Command("rm", "-rf", "generated")
+ c.Stdout = os.Stdout
+ c.Stderr = os.Stderr
+ c.Dir = genprotoDir
+ if err := c.Run(); err != nil {
+ return err
+ }
+
+ // Throw away changes to some special libs.
+ for _, lib := range []string{"googleapis/grafeas/v1", "googleapis/devtools/containeranalysis/v1"} {
+ c = exec.Command("git", "checkout", lib)
+ c.Stdout = os.Stdout
+ c.Stderr = os.Stderr
+ c.Dir = genprotoDir
+ if err := c.Run(); err != nil {
+ return err
+ }
+
+ c = exec.Command("git", "clean", "-df", lib)
+ c.Stdout = os.Stdout
+ c.Stderr = os.Stderr
+ c.Dir = genprotoDir
+ if err := c.Run(); err != nil {
+ return err
+ }
+ }
+
+ // Clean up and check it all compiles.
+ if err := vet(genprotoDir); err != nil {
+ return err
+ }
+
+ if err := build(genprotoDir); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+// goPkg reports the import path declared in the given file's `go_package`
+// option. If the option is missing, goPkg returns empty string.
+func goPkg(fname string) (string, error) {
+ content, err := ioutil.ReadFile(fname)
+ if err != nil {
+ return "", err
+ }
+
+ var pkgName string
+ if match := goPkgOptRe.FindSubmatch(content); len(match) > 0 {
+ pn, err := strconv.Unquote(string(match[1]))
+ if err != nil {
+ return "", err
+ }
+ pkgName = pn
+ }
+ if p := strings.IndexRune(pkgName, ';'); p > 0 {
+ pkgName = pkgName[:p]
+ }
+ return pkgName, nil
+}
+
+// protoc executes the "protoc" command on files named in fnames, and outputs
+// to "<genprotoDir>/generated".
+func protoc(genprotoDir, googleapisDir, protoDir string, fnames []string) error {
+ args := []string{fmt.Sprintf("--go_out=plugins=grpc:%s/generated", genprotoDir), "-I", googleapisDir, "-I", protoDir}
+ args = append(args, fnames...)
+ c := exec.Command("protoc", args...)
+ c.Stdout = os.Stdout
+ c.Stderr = os.Stderr
+ c.Dir = genprotoDir
+ return c.Run()
+}
diff --git a/internal/gapicgen/git.go b/internal/gapicgen/git.go
new file mode 100644
index 0000000..66b87ba
--- /dev/null
+++ b/internal/gapicgen/git.go
@@ -0,0 +1,74 @@
+// Copyright 2019 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package gapicgen provides some helpers for gapicgen binaries.
+package gapicgen
+
+import (
+ "fmt"
+ "os"
+ "os/exec"
+)
+
+// SetGitCreds sets credentials for gerrit.
+func SetGitCreds(githubName, githubEmail, gerritCookieName, gerritCookieValue string) error {
+ c := exec.Command("git", "config", "--global", "user.name", githubName)
+ c.Stdout = os.Stdout
+ c.Stderr = os.Stderr
+ c.Stdin = os.Stdin // Prevents "the input device is not a TTY" error.
+ c.Env = []string{
+ fmt.Sprintf("PATH=%s", os.Getenv("PATH")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands.
+ fmt.Sprintf("HOME=%s", os.Getenv("HOME")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands.
+ }
+ if err := c.Run(); err != nil {
+ return err
+ }
+
+ c = exec.Command("git", "config", "--global", "user.email", githubEmail)
+ c.Stdout = os.Stdout
+ c.Stderr = os.Stderr
+ c.Stdin = os.Stdin // Prevents "the input device is not a TTY" error.
+ c.Env = []string{
+ fmt.Sprintf("PATH=%s", os.Getenv("PATH")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands.
+ fmt.Sprintf("HOME=%s", os.Getenv("HOME")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands.
+ }
+ if err := c.Run(); err != nil {
+ return err
+ }
+
+ cmd := fmt.Sprintf(`
+set -ex
+
+eval 'set +o history' 2>/dev/null || setopt HIST_IGNORE_SPACE 2>/dev/null
+touch ~/.gitcookies
+chmod 0600 ~/.gitcookies
+
+git config --global http.cookiefile ~/.gitcookies
+
+tr , \\t <<\__END__ >>~/.gitcookies
+code.googlesource.com,FALSE,/,TRUE,2147483647,%s,%s
+code-review.googlesource.com,FALSE,/,TRUE,2147483647,%s,%s
+__END__
+eval 'set -o history' 2>/dev/null || unsetopt HIST_IGNORE_SPACE 2>/dev/null
+`, gerritCookieName, gerritCookieValue, gerritCookieName, gerritCookieValue)
+ c = exec.Command("/bin/bash", "-c", cmd)
+ c.Stdout = os.Stdout
+ c.Stderr = os.Stderr
+ c.Stdin = os.Stdin // Prevents "the input device is not a TTY" error.
+ c.Env = []string{
+ fmt.Sprintf("PATH=%s", os.Getenv("PATH")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands.
+ fmt.Sprintf("HOME=%s", os.Getenv("HOME")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands.
+ }
+ return c.Run()
+}
diff --git a/internal/gapicgen/go.mod b/internal/gapicgen/go.mod
new file mode 100644
index 0000000..808ea7c
--- /dev/null
+++ b/internal/gapicgen/go.mod
@@ -0,0 +1,12 @@
+module cloud.google.com/go/internal/gapicgen
+
+go 1.13
+
+require (
+ github.com/andygrunwald/go-gerrit v0.0.0-20191101112536-3f5e365ccf57
+ github.com/google/go-github v17.0.0+incompatible
+ github.com/google/go-querystring v1.0.0 // indirect
+ golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
+ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e
+ gopkg.in/src-d/go-git.v4 v4.13.1
+)
diff --git a/internal/gapicgen/go.sum b/internal/gapicgen/go.sum
new file mode 100644
index 0000000..8749f88
--- /dev/null
+++ b/internal/gapicgen/go.sum
@@ -0,0 +1,95 @@
+cloud.google.com/go v0.34.0 h1:eOI3/cP2VTU6uZLDYAoic+eyzzB9YyGmJ7eIjl8rOPg=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs=
+github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs=
+github.com/andygrunwald/go-gerrit v0.0.0-20191101112536-3f5e365ccf57 h1:wtSQ14h8qAUezER6QPfYmCh5+W5Ly1lVruhm/QeOVUE=
+github.com/andygrunwald/go-gerrit v0.0.0-20191101112536-3f5e365ccf57/go.mod h1:0iuRQp6WJ44ts+iihy5E/WlPqfg5RNeQxOmzRkxCdtk=
+github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA=
+github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
+github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
+github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
+github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg=
+github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
+github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ=
+github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
+github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0=
+github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
+github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
+github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
+github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
+github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
+github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
+github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
+github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
+github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd h1:Coekwdh0v2wtGp9Gmz1Ze3eVRAWJMLokvN3QjdzCHLY=
+github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo=
+github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
+github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
+github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4=
+github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
+github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70=
+github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4=
+golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc=
+golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 h1:Ao/3l156eZf2AW5wK8a7/smtodRU+gha3+BeqJ69lRk=
+golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e h1:D5TXcfTk7xF7hvieo4QErS3qqCB4teTffacDWr7CI+0=
+golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI=
+google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/src-d/go-billy.v4 v4.3.2 h1:0SQA1pRztfTFx2miS8sA97XvooFeNOmvUenF4o0EcVg=
+gopkg.in/src-d/go-billy.v4 v4.3.2/go.mod h1:nDjArDMp+XMs1aFAESLRjfGSgfvoYN0hDfzEk0GjC98=
+gopkg.in/src-d/go-git-fixtures.v3 v3.5.0 h1:ivZFOIltbce2Mo8IjzUHAFoq/IylO9WHhNOAJK+LsJg=
+gopkg.in/src-d/go-git-fixtures.v3 v3.5.0/go.mod h1:dLBcvytrw/TYZsNTWCnkNF2DSIlzWYqTe3rJR56Ac7g=
+gopkg.in/src-d/go-git.v4 v4.13.1 h1:SRtFyV8Kxc0UP7aCHcijOMQGPxHSmMOPrzulQWolkYE=
+gopkg.in/src-d/go-git.v4 v4.13.1/go.mod h1:nx5NYcxdKxq5fpltdHnPa2Exj4Sx0EclMWZQbYDu2z8=
+gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME=
+gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=
diff --git a/internal/gapicgen/tools.go b/internal/gapicgen/tools.go
new file mode 100644
index 0000000..3db3ee6
--- /dev/null
+++ b/internal/gapicgen/tools.go
@@ -0,0 +1,35 @@
+// Copyright 2019 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package gapicgen provides some helpers for gapicgen binaries.
+package gapicgen
+
+import (
+ "fmt"
+ "os"
+ "os/exec"
+)
+
+// VerifyAllToolsExist ensures that all required tools exist on the system.
+func VerifyAllToolsExist(toolsNeeded []string) error {
+ for _, t := range toolsNeeded {
+ c := exec.Command("which", t)
+ c.Stdout = os.Stdout
+ c.Stderr = os.Stderr
+ if c.Run() != nil {
+ return fmt.Errorf("%s does not appear to be installed. please install it. all tools needed: %v", t, toolsNeeded)
+ }
+ }
+ return nil
+}
diff --git a/internal/kokoro/vet.sh b/internal/kokoro/vet.sh
index 437b9df..b1c0d4b 100755
--- a/internal/kokoro/vet.sh
+++ b/internal/kokoro/vet.sh
@@ -77,6 +77,7 @@
grep -v "internal/testutil/funcmock.go" | \
grep -v "internal/backoff" | \
grep -v "internal/trace" | \
+ grep -v "internal/gapicgen/generator" | \
grep -v "a blank import should be only in a main or test package" | \
grep -v "method ExecuteSql should be ExecuteSQL" | \
grep -vE "spanner/spansql/(sql|types).go:.*should have comment" | \
diff --git a/manuals.txt b/manuals.txt
deleted file mode 100644
index 58b7bd1..0000000
--- a/manuals.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-errorreporting/apiv1beta1
-firestore/apiv1beta1
-firestore/apiv1
-logging/apiv2
-longrunning/autogen
-pubsub/apiv1
-spanner/apiv1
-trace/apiv1
diff --git a/microgens.csv b/microgens.csv
deleted file mode 100644
index 499119a..0000000
--- a/microgens.csv
+++ /dev/null
@@ -1,20 +0,0 @@
-input directory path, go module;package flag, gRPC ServiceConfig path flag, API service config path flag, release level
-google/cloud/texttospeech/v1, --go-gapic-package cloud.google.com/go/texttospeech/apiv1;texttospeech, --grpc-service-config google/cloud/texttospeech/v1/texttospeech_grpc_service_config.json, --gapic-service-config google/cloud/texttospeech/v1/texttospeech_v1.yaml, --release-level alpha
-google/cloud/asset/v1, --go-gapic-package cloud.google.com/go/asset/apiv1;asset, --grpc-service-config google/cloud/asset/v1/cloudasset_grpc_service_config.json, --gapic-service-config google/cloud/asset/v1/cloudasset_v1.yaml, --release-level alpha
-google/cloud/language/v1, --go-gapic-package cloud.google.com/go/language/apiv1;language, --grpc-service-config google/cloud/language/v1/language_grpc_service_config.json, --gapic-service-config google/cloud/language/language_v1.yaml, --release-level alpha
-google/cloud/phishingprotection/v1beta1, --go-gapic-package cloud.google.com/go/phishingprotection/apiv1beta1;phishingprotection, --grpc-service-config google/cloud/phishingprotection/v1beta1/phishingprotection_grpc_service_config.json, --gapic-service-config google/cloud/phishingprotection/v1beta1/phishingprotection_v1beta1.yaml, --release-level beta
-google/cloud/translate/v3, --go-gapic-package cloud.google.com/go/translate/apiv3;translate, --grpc-service-config google/cloud/translate/v3/translate_grpc_service_config.json, --gapic-service-config google/cloud/translate/v3/translate_v3.yaml,
-google/cloud/scheduler/v1, --go-gapic-package cloud.google.com/go/scheduler/apiv1;scheduler, --grpc-service-config google/cloud/scheduler/v1/cloudscheduler_grpc_service_config.json, --gapic-service-config google/cloud/scheduler/v1/cloudscheduler_v1.yaml,
-google/cloud/scheduler/v1beta1, --go-gapic-package cloud.google.com/go/scheduler/apiv1beta1;scheduler, --grpc-service-config google/cloud/scheduler/v1beta1/cloudscheduler_grpc_service_config.json, --gapic-service-config google/cloud/scheduler/v1beta1/cloudscheduler_v1beta1.yaml, --release-level beta
-google/cloud/speech/v1, --go-gapic-package cloud.google.com/go/speech/apiv1;speech, --grpc-service-config google/cloud/speech/v1/speech_grpc_service_config.json, --gapic-service-config google/cloud/speech/v1/speech_v1.yaml,
-google/cloud/speech/v1p1beta1, --go-gapic-package cloud.google.com/go/speech/apiv1p1beta1;speech, --grpc-service-config google/cloud/speech/v1p1beta1/speech_grpc_service_config.json, --gapic-service-config google/cloud/speech/v1p1beta1/speech_v1p1beta1.yaml, --release-level beta
-google/cloud/bigquery/datatransfer/v1, --go-gapic-package cloud.google.com/go/bigquery/datatransfer/apiv1;datatransfer, --grpc-service-config google/cloud/bigquery/datatransfer/v1/bigquerydatatransfer_grpc_service_config.json, --gapic-service-config google/cloud/bigquery/datatransfer/v1/bigquerydatatransfer_v1.yaml, --release-level alpha
-google/cloud/bigquery/storage/v1beta1, --go-gapic-package cloud.google.com/go/bigquery/storage/apiv1beta1;storage, --grpc-service-config google/cloud/bigquery/storage/v1beta1/bigquerystorage_grpc_service_config.json, --gapic-service-config google/cloud/bigquery/storage/v1beta1/bigquerystorage_v1beta1.yaml, --release-level beta
-google/cloud/iot/v1, --go-gapic-package cloud.google.com/go/iot/apiv1;iot, --grpc-service-config google/cloud/iot/v1/cloudiot_grpc_service_config.json, --gapic-service-config google/cloud/iot/v1/cloudiot_v1.yaml,
-google/cloud/recommender/v1beta1, --go-gapic-package cloud.google.com/go/recommender/apiv1beta1;recommender, --grpc-service-config google/cloud/recommender/v1beta1/recommender_grpc_service_config.json, --gapic-service-config google/cloud/recommender/v1beta1/recommender_v1beta1.yaml, --release-level beta
-google/cloud/tasks/v2, --go-gapic-package cloud.google.com/go/cloudtasks/apiv2;cloudtasks, --grpc-service-config google/cloud/tasks/v2/cloudtasks_grpc_service_config.json, --gapic-service-config google/cloud/tasks/v2/cloudtasks_v2.yaml,
-google/cloud/tasks/v2beta2, --go-gapic-package cloud.google.com/go/cloudtasks/apiv2beta2;cloudtasks, --grpc-service-config google/cloud/tasks/v2beta2/cloudtasks_grpc_service_config.json, --gapic-service-config google/cloud/tasks/v2beta2/cloudtasks_v2beta2.yaml, --release-level beta
-google/cloud/tasks/v2beta3, --go-gapic-package cloud.google.com/go/cloudtasks/apiv2beta3;cloudtasks, --grpc-service-config google/cloud/tasks/v2beta3/cloudtasks_grpc_service_config.json, --gapic-service-config google/cloud/tasks/v2beta3/cloudtasks_v2beta3.yaml, --release-level beta
-google/cloud/videointelligence/v1, --go-gapic-package cloud.google.com/go/videointelligence/apiv1;videointelligence, --grpc-service-config google/cloud/videointelligence/v1/videointelligence_grpc_service_config.json, --gapic-service-config google/cloud/videointelligence/v1/videointelligence_v1.yaml, --release-level alpha
-google/cloud/vision/v1, --go-gapic-package cloud.google.com/go/vision/apiv1;vision, --grpc-service-config google/cloud/vision/v1/vision_grpc_service_config.json, --gapic-service-config google/cloud/vision/v1/vision_v1.yaml,
-google/cloud/webrisk/v1beta1, --go-gapic-package cloud.google.com/go/webrisk/apiv1beta1;webrisk, --grpc-service-config google/cloud/webrisk/v1beta1/webrisk_grpc_service_config.json, --gapic-service-config google/cloud/webrisk/v1beta1/webrisk_v1beta1.yaml, --release-level beta
diff --git a/regen-gapic.sh b/regen-gapic.sh
deleted file mode 100755
index f00eb1a..0000000
--- a/regen-gapic.sh
+++ /dev/null
@@ -1,81 +0,0 @@
-#!/bin/bash
-# Copyright 2019 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# This script generates all GAPIC clients in this repo.
-# See instructions at go/yoshi-site.
-
-set -ex
-
-GOCLOUD_DIR="$(dirname "$0")"
-HOST_MOUNT="$PWD"
-
-# need to mount the /var/folders properly for macos
-# https://stackoverflow.com/questions/45122459/docker-mounts-denied-the-paths-are-not-shared-from-os-x-and-are-not-known/45123074
-if [[ "$OSTYPE" == "darwin"* ]] && [[ "$HOST_MOUNT" == "/var/folders"* ]]; then
- HOST_MOUNT=/private$HOST_MOUNT
-fi
-
-microgen() {
- input=$1
- options="${@:2}"
-
- # see https://github.com/googleapis/gapic-generator-go/blob/master/README.md#docker-wrapper for details
- docker run \
- --user $UID \
- --mount type=bind,source=$HOST_MOUNT,destination=/conf,readonly \
- --mount type=bind,source=$HOST_MOUNT/$input,destination=/in/$input,readonly \
- --mount type=bind,source=/tmp,destination=/out \
- --rm \
- gcr.io/gapic-images/gapic-generator-go:0.9.5 \
- $options
-}
-
-for gencfg in $(cat $GOCLOUD_DIR/gapics.txt); do
- rm -rf artman-genfiles/*
- artman --config "$gencfg" generate go_gapic
- cp -r artman-genfiles/gapi-*/cloud.google.com/go/* $GOCLOUD_DIR
-done
-
-rm -rf /tmp/cloud.google.com
-{
- # skip the first line with column titles
- read -r
- while IFS=, read -r input mod retrycfg apicfg release
- do
- microgen $input "$mod" "$retrycfg" "$apicfg" "$release"
- done
-} < $GOCLOUD_DIR/microgens.csv
-
-# copy generated code if any was created
-[ -d "/tmp/cloud.google.com/go" ] && cp -r /tmp/cloud.google.com/go/* $GOCLOUD_DIR
-
-pushd $GOCLOUD_DIR
- gofmt -s -d -l -w . && goimports -w .
-
- # NOTE(pongad): `sed -i` doesn't work on Macs, because -i option needs an argument.
- # `-i ''` doesn't work on GNU, since the empty string is treated as a file name.
- # So we just create the backup and delete it after.
- ver=$(date +%Y%m%d)
- git ls-files -mo | while read modified; do
- dir=${modified%/*.*}
- find . -path "*/$dir/doc.go" -exec sed -i.backup -e "s/^const versionClient.*/const versionClient = \"$ver\"/" '{}' +
- done
-popd
-
-for manualdir in $(cat $GOCLOUD_DIR/manuals.txt); do
- find "$GOCLOUD_DIR/$manualdir" -name '*.go' -exec sed -i.backup -e 's/setGoogleClientInfo/SetGoogleClientInfo/g' '{}' '+'
-done
-
-find $GOCLOUD_DIR -name '*.backup' -delete
diff --git a/tidyall.sh b/tidyall.sh
deleted file mode 100755
index e9bc9d6..0000000
--- a/tidyall.sh
+++ /dev/null
@@ -1,28 +0,0 @@
-#!/bin/bash
-# Copyright 2019 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Run at repo root.
-
-go mod tidy
-for i in `find . -name go.mod`; do
-pushd `dirname $i`;
- # Update genproto and api to latest for every module (latest version is
- # always correct version). tidy will remove the dependencies if they're not
- # actually used by the client.
- go get -u google.golang.org/api
- go get -u google.golang.org/genproto
- go mod tidy;
-popd;
-done
