profiler/proftest/proftest.go - gocloud - Git at Google

 // Copyright 2018 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //      http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
 //
 // Package proftest contains test helpers for profiler agent integration tests.
 // This package is experimental.

 package proftest

 import (
 	"bytes"
 	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"io/ioutil"
 	"log"
 	"net/http"
 	"strings"
 	"text/template"
 	"time"

 	"cloud.google.com/go/storage"
 	gax "github.com/googleapis/gax-go/v2"
 	compute "google.golang.org/api/compute/v1"
 	container "google.golang.org/api/container/v1"
 	"google.golang.org/api/googleapi"
 )

 const (
 	monitorWriteScope = "https://www.googleapis.com/auth/monitoring.write"
 )

 // BaseStartupTmpl is the common part of the startup script that
 // can be shared by multiple tests.
 var BaseStartupTmpl = template.Must(template.New("startupScript").Parse(`
 {{ define "prologue" -}}
 #! /bin/bash
 (
 # Signal any unexpected error.
 trap 'echo "{{.ErrorString}}"' ERR

 # Shut down the VM in 5 minutes after this script exits
 # to stop accounting the VM for billing and cores quota.
 trap "sleep 300 && poweroff" EXIT

 retry() {
   for i in {1..3}; do
     "${@}" && return 0
   done
   return 1
 }

 # Fail on any error.
 set -eo pipefail

 # Display commands being run
 set -x
 {{- end }}

 {{ define "epilogue" -}}
 # Write output to serial port 2 with timestamp.
 ) 2>&1 | while read line; do echo "$(date): ${line}"; done >/dev/ttyS1
 {{- end }}
 `))

 // TestRunner has common elements used for testing profiling agents on a range
 // of environments.
 type TestRunner struct {
 	Client *http.Client
 }

 // GCETestRunner supports testing a profiling agent on GCE.
 type GCETestRunner struct {
 	TestRunner
 	ComputeService *compute.Service
 }

 // GKETestRunner supports testing a profiling agent on GKE.
 type GKETestRunner struct {
 	TestRunner
 	ContainerService *container.Service
 	StorageClient    *storage.Client
 	Dockerfile       string
 }

 // ProfileResponse contains the response produced when querying profile server.
 type ProfileResponse struct {
 	Profile     ProfileData   `json:"profile"`
 	NumProfiles int32         `json:"numProfiles"`
 	Deployments []interface{} `json:"deployments"`
 }

 // ProfileData has data of a single profile.
 type ProfileData struct {
 	Samples           []int32         `json:"samples"`
 	SampleMetrics     interface{}     `json:"sampleMetrics"`
 	DefaultMetricType string          `json:"defaultMetricType"`
 	TreeNodes         interface{}     `json:"treeNodes"`
 	Functions         functionArray   `json:"functions"`
 	SourceFiles       sourceFileArray `json:"sourceFiles"`
 }

 type functionArray struct {
 	Name       []string `json:"name"`
 	Sourcefile []int32  `json:"sourceFile"`
 }

 type sourceFileArray struct {
 	Name []string `json:"name"`
 }

 // InstanceConfig is configuration for starting single GCE instance for
 // profiling agent test case.
 type InstanceConfig struct {
 	ProjectID     string
 	Zone          string
 	Name          string
 	StartupScript string
 	MachineType   string
 	ImageProject  string
 	ImageFamily   string
 	Scopes        []string
 }

 // ClusterConfig is configuration for starting single GKE cluster for profiling
 // agent test case.
 type ClusterConfig struct {
 	ProjectID       string
 	Zone            string
 	ClusterName     string
 	PodName         string
 	ImageSourceName string
 	ImageName       string
 	Bucket          string
 	Dockerfile      string
 }

 // queryProfileRequest is the request format for querying the profile server.
 type queryProfileRequest struct {
 	StartTime        string            `json:"startTime"`
 	EndTime          string            `json:"endTime"`
 	ProfileType      string            `json:"profileType"`
 	Target           string            `json:"target"`
 	DeploymentLabels map[string]string `json:"deploymentLabels,omitempty"`
 }

 // CheckNonEmpty returns nil if the profile has a profiles and deployments
 // associated. Otherwise, returns a desciptive error.
 func (pr *ProfileResponse) CheckNonEmpty() error {
 	if pr.NumProfiles == 0 {
 		return fmt.Errorf("profile response contains zero profiles: %v", pr)
 	}
 	if len(pr.Deployments) == 0 {
 		return fmt.Errorf("profile response contains zero deployments: %v", pr)
 	}
 	return nil
 }

 // HasFunction returns nil if the function is present, or, if the function is
 // not present, and error providing more details why the function is not
 // present.
 func (pr *ProfileResponse) HasFunction(functionName string) error {
 	if err := pr.CheckNonEmpty(); err != nil {
 		return fmt.Errorf("failed to find function name %s in profile: %v", functionName, err)
 	}
 	for _, name := range pr.Profile.Functions.Name {
 		if strings.Contains(name, functionName) {
 			return nil
 		}
 	}
 	return fmt.Errorf("failed to find function name %s in profile", functionName)
 }

 // HasFunctionInFile returns nil if function is present in the specifed file, and an
 // error if the function/file combination is not present in the profile.
 func (pr *ProfileResponse) HasFunctionInFile(functionName string, filename string) error {
 	if err := pr.CheckNonEmpty(); err != nil {
 		return fmt.Errorf("failed to find function name %s in file %s in profile: %v", functionName, filename, err)
 	}
 	for i, name := range pr.Profile.Functions.Name {
 		file := pr.Profile.SourceFiles.Name[pr.Profile.Functions.Sourcefile[i]]
 		if strings.Contains(name, functionName) && strings.HasSuffix(file, filename) {
 			return nil
 		}
 	}
 	return fmt.Errorf("failed to find function name %s in file %s in profile", functionName, filename)
 }

 // HasSourceFile returns nil if the file (or file where the end of the file path
 // matches the filename) is present in the profile. Or, if the filename is not
 // present, an error is returned.
 func (pr *ProfileResponse) HasSourceFile(filename string) error {
 	if err := pr.CheckNonEmpty(); err != nil {
 		return fmt.Errorf("failed to find filename %s in profile: %v", filename, err)
 	}
 	for _, name := range pr.Profile.SourceFiles.Name {
 		if strings.HasSuffix(name, filename) {
 			return nil
 		}
 	}
 	return fmt.Errorf("failed to find filename %s in profile", filename)
 }

 // StartInstance starts a GCE Instance with configs specified by inst,
 // and which runs the startup script specified in inst. If image project
 // is not specified, it defaults to "debian-cloud". If image family is
 // not specified, it defaults to "debian-9".
 func (tr *GCETestRunner) StartInstance(ctx context.Context, inst *InstanceConfig) error {
 	imageProject, imageFamily := inst.ImageProject, inst.ImageFamily
 	if imageProject == "" {
 		imageProject = "debian-cloud"
 	}
 	if imageFamily == "" {
 		imageFamily = "debian-9"
 	}
 	img, err := tr.ComputeService.Images.GetFromFamily(imageProject, imageFamily).Context(ctx).Do()
 	if err != nil {
 		return fmt.Errorf("failed to get image from family %q in project %q: %v", imageFamily, imageProject, err)
 	}

 	op, err := tr.ComputeService.Instances.Insert(inst.ProjectID, inst.Zone, &compute.Instance{
 		MachineType: fmt.Sprintf("zones/%s/machineTypes/%s", inst.Zone, inst.MachineType),
 		Name:        inst.Name,
 		Disks: []*compute.AttachedDisk{{
 			AutoDelete: true, // delete the disk when the VM is deleted.
 			Boot:       true,
 			Type:       "PERSISTENT",
 			Mode:       "READ_WRITE",
 			InitializeParams: &compute.AttachedDiskInitializeParams{
 				SourceImage: img.SelfLink,
 				DiskType:    fmt.Sprintf("https://www.googleapis.com/compute/v1/projects/%s/zones/%s/diskTypes/pd-standard", inst.ProjectID, inst.Zone),
 			},
 		}},
 		NetworkInterfaces: []*compute.NetworkInterface{{
 			Network: fmt.Sprintf("https://www.googleapis.com/compute/v1/projects/%s/global/networks/default", inst.ProjectID),
 			AccessConfigs: []*compute.AccessConfig{{
 				Name: "External NAT",
 			}},
 		}},
 		Metadata: &compute.Metadata{
 			Items: []*compute.MetadataItems{{
 				Key:   "startup-script",
 				Value: googleapi.String(inst.StartupScript),
 			}},
 		},
 		ServiceAccounts: []*compute.ServiceAccount{{
 			Email:  "default",
 			Scopes: append(inst.Scopes, monitorWriteScope),
 		}},
 	}).Do()

 	if err != nil {
 		return fmt.Errorf("failed to create instance: %v", err)
 	}

 	// Poll status of the operation to create the instance.
 	getOpCall := tr.ComputeService.ZoneOperations.Get(inst.ProjectID, inst.Zone, op.Name)
 	for {
 		if err := checkOpErrors(op); err != nil {
 			return fmt.Errorf("failed to create instance: %v", err)
 		}
 		if op.Status == "DONE" {
 			return nil
 		}

 		if err := gax.Sleep(ctx, 5*time.Second); err != nil {
 			return err
 		}

 		op, err = getOpCall.Do()
 		if err != nil {
 			return fmt.Errorf("failed to get operation: %v", err)
 		}
 	}
 }

 // checkOpErrors returns nil if the operation does not have any errors and an
 // error summarizing all errors encountered if the operation has errored.
 func checkOpErrors(op *compute.Operation) error {
 	if op.Error == nil || len(op.Error.Errors) == 0 {
 		return nil
 	}

 	var errs []string
 	for _, e := range op.Error.Errors {
 		if e.Message != "" {
 			errs = append(errs, e.Message)
 		} else {
 			errs = append(errs, e.Code)
 		}
 	}
 	return errors.New(strings.Join(errs, ","))
 }

 // DeleteInstance deletes an instance with project id, name, and zone matched
 // by inst.
 func (tr *GCETestRunner) DeleteInstance(ctx context.Context, inst *InstanceConfig) error {
 	if _, err := tr.ComputeService.Instances.Delete(inst.ProjectID, inst.Zone, inst.Name).Context(ctx).Do(); err != nil {
 		return fmt.Errorf("Instances.Delete(%s) got error: %v", inst.Name, err)
 	}
 	return nil
 }

 // PollForSerialOutput polls serial port 2 of the GCE instance specified by
 // inst and returns when the finishString appears in the serial output
 // of the instance, or when the context times out.
 func (tr *GCETestRunner) PollForSerialOutput(ctx context.Context, inst *InstanceConfig, finishString, errorString string) error {
 	var output string
 	defer func() {
 		log.Printf("Serial port output for %s:\n%s", inst.Name, output)
 	}()

 	for {
 		select {
 		case <-ctx.Done():
 			return ctx.Err()
 		case <-time.After(20 * time.Second):
 			resp, err := tr.ComputeService.Instances.GetSerialPortOutput(inst.ProjectID, inst.Zone, inst.Name).Port(2).Context(ctx).Do()
 			if err != nil {
 				// Transient failure.
 				log.Printf("Transient error getting serial port output from instance %s (will retry): %v", inst.Name, err)
 				continue
 			}
 			if resp.Contents == "" {
 				log.Printf("Ignoring empty serial port output from instance %s (will retry)", inst.Name)
 				continue
 			}
 			if output = resp.Contents; strings.Contains(output, finishString) {
 				return nil
 			}
 			if strings.Contains(output, errorString) {
 				return fmt.Errorf("failed to execute the prober benchmark script")
 			}
 		}
 	}
 }

 // QueryProfiles retrieves profiles of a specific type, from a specific time
 // range, associated with a particular service and project.
 func (tr *TestRunner) QueryProfiles(projectID, service, startTime, endTime, profileType string) (ProfileResponse, error) {
 	return tr.QueryProfilesWithZone(projectID, service, startTime, endTime, profileType, "")
 }

 // QueryProfilesWithZone retrieves profiles of a specific type, from a specific
 // time range, in a specified zone, associated with a particular service
 // and project.
 func (tr *TestRunner) QueryProfilesWithZone(projectID, service, startTime, endTime, profileType, zone string) (ProfileResponse, error) {
 	queryURL := fmt.Sprintf("https://cloudprofiler.googleapis.com/v2/projects/%s/profiles:query", projectID)
 	deploymentLabels := map[string]string{}
 	if zone != "" {
 		deploymentLabels["zone"] = zone
 	}

 	qpr := queryProfileRequest{
 		StartTime:        startTime,
 		EndTime:          endTime,
 		ProfileType:      profileType,
 		Target:           service,
 		DeploymentLabels: deploymentLabels,
 	}

 	queryJSON, err := json.Marshal(qpr)
 	if err != nil {
 		return ProfileResponse{}, fmt.Errorf("failed to marshall request to JSON: %v", err)
 	}

 	req, err := http.NewRequest("POST", queryURL, bytes.NewReader(queryJSON))
 	if err != nil {
 		return ProfileResponse{}, fmt.Errorf("failed to create an API request: %v", err)
 	}
 	req.Header = map[string][]string{
 		"X-Goog-User-Project": {projectID},
 	}

 	resp, err := tr.Client.Do(req)
 	if err != nil {
 		return ProfileResponse{}, fmt.Errorf("failed to query API: %v", err)
 	}
 	defer resp.Body.Close()

 	body, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
 		return ProfileResponse{}, fmt.Errorf("failed to read response body: %v", err)
 	}

 	if resp.StatusCode != 200 {
 		return ProfileResponse{}, fmt.Errorf("failed to query API: status: %s, response body: %s", resp.Status, string(body))
 	}

 	var pr ProfileResponse
 	if err := json.Unmarshal(body, &pr); err != nil {
 		return ProfileResponse{}, err
 	}

 	return pr, nil
 }

 type imageResponse struct {
 	Manifest map[string]interface{} `json:"manifest"`
 	Name     string                 `json:"name"`
 	Tags     []string               `json:"tags"`
 }

 // deleteDockerImage deletes a docker image from Google Container Registry.
 func (tr *GKETestRunner) deleteDockerImage(ctx context.Context, ImageName string) []error {
 	queryImageURL := fmt.Sprintf("https://gcr.io/v2/%s/tags/list", ImageName)
 	resp, err := tr.Client.Get(queryImageURL)
 	if err != nil {
 		return []error{fmt.Errorf("failed to list tags: %v", err)}
 	}
 	defer resp.Body.Close()

 	body, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
 		return []error{err}
 	}
 	var ir imageResponse
 	if err := json.Unmarshal(body, &ir); err != nil {
 		return []error{err}
 	}

 	const deleteImageURLFmt = "https://gcr.io/v2/%s/manifests/%s"
 	var errs []error
 	for _, tag := range ir.Tags {
 		if err := deleteDockerImageResource(tr.Client, fmt.Sprintf(deleteImageURLFmt, ImageName, tag)); err != nil {
 			errs = append(errs, fmt.Errorf("failed to delete tag %s: %v", tag, err))
 		}
 	}

 	for manifest := range ir.Manifest {
 		if err := deleteDockerImageResource(tr.Client, fmt.Sprintf(deleteImageURLFmt, ImageName, manifest)); err != nil {
 			errs = append(errs, fmt.Errorf("failed to delete manifest %s: %v", manifest, err))
 		}
 	}
 	return errs
 }

 func deleteDockerImageResource(client *http.Client, url string) error {
 	req, err := http.NewRequest("DELETE", url, nil)
 	if err != nil {
 		return fmt.Errorf("failed to get request: %v", err)
 	}
 	resp, err := client.Do(req)
 	if err != nil {
 		return fmt.Errorf("failed to delete resource: %v", err)
 	}
 	defer resp.Body.Close()
 	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted {
 		return fmt.Errorf("failed to delete resource: status code = %d", resp.StatusCode)
 	}
 	return nil
 }

 // DeleteClusterAndImage deletes cluster and images used to create cluster.
 func (tr *GKETestRunner) DeleteClusterAndImage(ctx context.Context, cfg *ClusterConfig) []error {
 	var errs []error
 	if err := tr.StorageClient.Bucket(cfg.Bucket).Object(cfg.ImageSourceName).Delete(ctx); err != nil {
 		errs = append(errs, fmt.Errorf("failed to delete storage client: %v", err))
 	}
 	for _, err := range tr.deleteDockerImage(ctx, cfg.ImageName) {
 		errs = append(errs, fmt.Errorf("failed to delete docker image: %v", err))
 	}
 	if _, err := tr.ContainerService.Projects.Zones.Clusters.Delete(cfg.ProjectID, cfg.Zone, cfg.ClusterName).Context(ctx).Do(); err != nil {
 		errs = append(errs, fmt.Errorf("failed to delete cluster %s: %v", cfg.ClusterName, err))
 	}

 	return errs
 }
	// Copyright 2018 Google LLC
	//
	// Licensed under the Apache License, Version 2.0 (the "License");
	// you may not use this file except in compliance with the License.
	// You may obtain a copy of the License at
	//
	// http://www.apache.org/licenses/LICENSE-2.0
	//
	// Unless required by applicable law or agreed to in writing, software
	// distributed under the License is distributed on an "AS IS" BASIS,
	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	// See the License for the specific language governing permissions and
	// limitations under the License.
	//
	// Package proftest contains test helpers for profiler agent integration tests.
	// This package is experimental.

	package proftest

	import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"strings"
	"text/template"
	"time"

	"cloud.google.com/go/storage"
	gax "github.com/googleapis/gax-go/v2"
	compute "google.golang.org/api/compute/v1"
	container "google.golang.org/api/container/v1"
	"google.golang.org/api/googleapi"
	)

	const (
	monitorWriteScope = "https://www.googleapis.com/auth/monitoring.write"
	)

	// BaseStartupTmpl is the common part of the startup script that
	// can be shared by multiple tests.
	var BaseStartupTmpl = template.Must(template.New("startupScript").Parse(`
	{{ define "prologue" -}}
	#! /bin/bash
	(
	# Signal any unexpected error.
	trap 'echo "{{.ErrorString}}"' ERR

	# Shut down the VM in 5 minutes after this script exits
	# to stop accounting the VM for billing and cores quota.
	trap "sleep 300 && poweroff" EXIT

	retry() {
	for i in {1..3}; do
	"${@}" && return 0
	done
	return 1
	}

	# Fail on any error.
	set -eo pipefail

	# Display commands being run
	set -x
	{{- end }}

	{{ define "epilogue" -}}
	# Write output to serial port 2 with timestamp.
	) 2>&1 \| while read line; do echo "$(date): ${line}"; done >/dev/ttyS1
	{{- end }}
	`))

	// TestRunner has common elements used for testing profiling agents on a range
	// of environments.
	type TestRunner struct {
	Client *http.Client
	}

	// GCETestRunner supports testing a profiling agent on GCE.
	type GCETestRunner struct {
	TestRunner
	ComputeService *compute.Service
	}

	// GKETestRunner supports testing a profiling agent on GKE.
	type GKETestRunner struct {
	TestRunner
	ContainerService *container.Service
	StorageClient *storage.Client
	Dockerfile string
	}

	// ProfileResponse contains the response produced when querying profile server.
	type ProfileResponse struct {
	Profile ProfileData `json:"profile"`
	NumProfiles int32 `json:"numProfiles"`
	Deployments []interface{} `json:"deployments"`
	}

	// ProfileData has data of a single profile.
	type ProfileData struct {
	Samples []int32 `json:"samples"`
	SampleMetrics interface{} `json:"sampleMetrics"`
	DefaultMetricType string `json:"defaultMetricType"`
	TreeNodes interface{} `json:"treeNodes"`
	Functions functionArray `json:"functions"`
	SourceFiles sourceFileArray `json:"sourceFiles"`
	}

	type functionArray struct {
	Name []string `json:"name"`
	Sourcefile []int32 `json:"sourceFile"`
	}

	type sourceFileArray struct {
	Name []string `json:"name"`
	}

	// InstanceConfig is configuration for starting single GCE instance for
	// profiling agent test case.
	type InstanceConfig struct {
	ProjectID string
	Zone string
	Name string
	StartupScript string
	MachineType string
	ImageProject string
	ImageFamily string
	Scopes []string
	}

	// ClusterConfig is configuration for starting single GKE cluster for profiling
	// agent test case.
	type ClusterConfig struct {
	ProjectID string
	Zone string
	ClusterName string
	PodName string
	ImageSourceName string
	ImageName string
	Bucket string
	Dockerfile string
	}

	// queryProfileRequest is the request format for querying the profile server.
	type queryProfileRequest struct {
	StartTime string `json:"startTime"`
	EndTime string `json:"endTime"`
	ProfileType string `json:"profileType"`
	Target string `json:"target"`
	DeploymentLabels map[string]string `json:"deploymentLabels,omitempty"`
	}

	// CheckNonEmpty returns nil if the profile has a profiles and deployments
	// associated. Otherwise, returns a desciptive error.
	func (pr *ProfileResponse) CheckNonEmpty() error {
	if pr.NumProfiles == 0 {
	return fmt.Errorf("profile response contains zero profiles: %v", pr)
	}
	if len(pr.Deployments) == 0 {
	return fmt.Errorf("profile response contains zero deployments: %v", pr)
	}
	return nil
	}

	// HasFunction returns nil if the function is present, or, if the function is
	// not present, and error providing more details why the function is not
	// present.
	func (pr *ProfileResponse) HasFunction(functionName string) error {
	if err := pr.CheckNonEmpty(); err != nil {
	return fmt.Errorf("failed to find function name %s in profile: %v", functionName, err)
	}
	for _, name := range pr.Profile.Functions.Name {
	if strings.Contains(name, functionName) {
	return nil
	}
	}
	return fmt.Errorf("failed to find function name %s in profile", functionName)
	}

	// HasFunctionInFile returns nil if function is present in the specifed file, and an
	// error if the function/file combination is not present in the profile.
	func (pr *ProfileResponse) HasFunctionInFile(functionName string, filename string) error {
	if err := pr.CheckNonEmpty(); err != nil {
	return fmt.Errorf("failed to find function name %s in file %s in profile: %v", functionName, filename, err)
	}
	for i, name := range pr.Profile.Functions.Name {
	file := pr.Profile.SourceFiles.Name[pr.Profile.Functions.Sourcefile[i]]
	if strings.Contains(name, functionName) && strings.HasSuffix(file, filename) {
	return nil
	}
	}
	return fmt.Errorf("failed to find function name %s in file %s in profile", functionName, filename)
	}

	// HasSourceFile returns nil if the file (or file where the end of the file path
	// matches the filename) is present in the profile. Or, if the filename is not
	// present, an error is returned.
	func (pr *ProfileResponse) HasSourceFile(filename string) error {
	if err := pr.CheckNonEmpty(); err != nil {
	return fmt.Errorf("failed to find filename %s in profile: %v", filename, err)
	}
	for _, name := range pr.Profile.SourceFiles.Name {
	if strings.HasSuffix(name, filename) {
	return nil
	}
	}
	return fmt.Errorf("failed to find filename %s in profile", filename)
	}

	// StartInstance starts a GCE Instance with configs specified by inst,
	// and which runs the startup script specified in inst. If image project
	// is not specified, it defaults to "debian-cloud". If image family is
	// not specified, it defaults to "debian-9".
	func (tr GCETestRunner) StartInstance(ctx context.Context, inst InstanceConfig) error {
	imageProject, imageFamily := inst.ImageProject, inst.ImageFamily
	if imageProject == "" {
	imageProject = "debian-cloud"
	}
	if imageFamily == "" {
	imageFamily = "debian-9"
	}
	img, err := tr.ComputeService.Images.GetFromFamily(imageProject, imageFamily).Context(ctx).Do()
	if err != nil {
	return fmt.Errorf("failed to get image from family %q in project %q: %v", imageFamily, imageProject, err)
	}

	op, err := tr.ComputeService.Instances.Insert(inst.ProjectID, inst.Zone, &compute.Instance{
	MachineType: fmt.Sprintf("zones/%s/machineTypes/%s", inst.Zone, inst.MachineType),
	Name: inst.Name,
	Disks: []*compute.AttachedDisk{{
	AutoDelete: true, // delete the disk when the VM is deleted.
	Boot: true,
	Type: "PERSISTENT",
	Mode: "READ_WRITE",
	InitializeParams: &compute.AttachedDiskInitializeParams{
	SourceImage: img.SelfLink,
	DiskType: fmt.Sprintf("https://www.googleapis.com/compute/v1/projects/%s/zones/%s/diskTypes/pd-standard", inst.ProjectID, inst.Zone),
	},
	}},
	NetworkInterfaces: []*compute.NetworkInterface{{
	Network: fmt.Sprintf("https://www.googleapis.com/compute/v1/projects/%s/global/networks/default", inst.ProjectID),
	AccessConfigs: []*compute.AccessConfig{{
	Name: "External NAT",
	}},
	}},
	Metadata: &compute.Metadata{
	Items: []*compute.MetadataItems{{
	Key: "startup-script",
	Value: googleapi.String(inst.StartupScript),
	}},
	},
	ServiceAccounts: []*compute.ServiceAccount{{
	Email: "default",
	Scopes: append(inst.Scopes, monitorWriteScope),
	}},
	}).Do()

	if err != nil {
	return fmt.Errorf("failed to create instance: %v", err)
	}

	// Poll status of the operation to create the instance.
	getOpCall := tr.ComputeService.ZoneOperations.Get(inst.ProjectID, inst.Zone, op.Name)
	for {
	if err := checkOpErrors(op); err != nil {
	return fmt.Errorf("failed to create instance: %v", err)
	}
	if op.Status == "DONE" {
	return nil
	}

	if err := gax.Sleep(ctx, 5*time.Second); err != nil {
	return err
	}

	op, err = getOpCall.Do()
	if err != nil {
	return fmt.Errorf("failed to get operation: %v", err)
	}
	}
	}

	// checkOpErrors returns nil if the operation does not have any errors and an
	// error summarizing all errors encountered if the operation has errored.
	func checkOpErrors(op *compute.Operation) error {
	if op.Error == nil \|\| len(op.Error.Errors) == 0 {
	return nil
	}

	var errs []string
	for _, e := range op.Error.Errors {
	if e.Message != "" {
	errs = append(errs, e.Message)
	} else {
	errs = append(errs, e.Code)
	}
	}
	return errors.New(strings.Join(errs, ","))
	}

	// DeleteInstance deletes an instance with project id, name, and zone matched
	// by inst.
	func (tr GCETestRunner) DeleteInstance(ctx context.Context, inst InstanceConfig) error {
	if _, err := tr.ComputeService.Instances.Delete(inst.ProjectID, inst.Zone, inst.Name).Context(ctx).Do(); err != nil {
	return fmt.Errorf("Instances.Delete(%s) got error: %v", inst.Name, err)
	}
	return nil
	}

	// PollForSerialOutput polls serial port 2 of the GCE instance specified by
	// inst and returns when the finishString appears in the serial output
	// of the instance, or when the context times out.
	func (tr GCETestRunner) PollForSerialOutput(ctx context.Context, inst InstanceConfig, finishString, errorString string) error {
	var output string
	defer func() {
	log.Printf("Serial port output for %s:\n%s", inst.Name, output)
	}()

	for {
	select {
	case <-ctx.Done():
	return ctx.Err()
	case <-time.After(20 * time.Second):
	resp, err := tr.ComputeService.Instances.GetSerialPortOutput(inst.ProjectID, inst.Zone, inst.Name).Port(2).Context(ctx).Do()
	if err != nil {
	// Transient failure.
	log.Printf("Transient error getting serial port output from instance %s (will retry): %v", inst.Name, err)
	continue
	}
	if resp.Contents == "" {
	log.Printf("Ignoring empty serial port output from instance %s (will retry)", inst.Name)
	continue
	}
	if output = resp.Contents; strings.Contains(output, finishString) {
	return nil
	}
	if strings.Contains(output, errorString) {
	return fmt.Errorf("failed to execute the prober benchmark script")
	}
	}
	}
	}

	// QueryProfiles retrieves profiles of a specific type, from a specific time
	// range, associated with a particular service and project.
	func (tr *TestRunner) QueryProfiles(projectID, service, startTime, endTime, profileType string) (ProfileResponse, error) {
	return tr.QueryProfilesWithZone(projectID, service, startTime, endTime, profileType, "")
	}

	// QueryProfilesWithZone retrieves profiles of a specific type, from a specific
	// time range, in a specified zone, associated with a particular service
	// and project.
	func (tr *TestRunner) QueryProfilesWithZone(projectID, service, startTime, endTime, profileType, zone string) (ProfileResponse, error) {
	queryURL := fmt.Sprintf("https://cloudprofiler.googleapis.com/v2/projects/%s/profiles:query", projectID)
	deploymentLabels := map[string]string{}
	if zone != "" {
	deploymentLabels["zone"] = zone
	}

	qpr := queryProfileRequest{
	StartTime: startTime,
	EndTime: endTime,
	ProfileType: profileType,
	Target: service,
	DeploymentLabels: deploymentLabels,
	}

	queryJSON, err := json.Marshal(qpr)
	if err != nil {
	return ProfileResponse{}, fmt.Errorf("failed to marshall request to JSON: %v", err)
	}

	req, err := http.NewRequest("POST", queryURL, bytes.NewReader(queryJSON))
	if err != nil {
	return ProfileResponse{}, fmt.Errorf("failed to create an API request: %v", err)
	}
	req.Header = map[string][]string{
	"X-Goog-User-Project": {projectID},
	}

	resp, err := tr.Client.Do(req)
	if err != nil {
	return ProfileResponse{}, fmt.Errorf("failed to query API: %v", err)
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
	return ProfileResponse{}, fmt.Errorf("failed to read response body: %v", err)
	}

	if resp.StatusCode != 200 {
	return ProfileResponse{}, fmt.Errorf("failed to query API: status: %s, response body: %s", resp.Status, string(body))
	}

	var pr ProfileResponse
	if err := json.Unmarshal(body, &pr); err != nil {
	return ProfileResponse{}, err
	}

	return pr, nil
	}

	type imageResponse struct {
	Manifest map[string]interface{} `json:"manifest"`
	Name string `json:"name"`
	Tags []string `json:"tags"`
	}

	// deleteDockerImage deletes a docker image from Google Container Registry.
	func (tr *GKETestRunner) deleteDockerImage(ctx context.Context, ImageName string) []error {
	queryImageURL := fmt.Sprintf("https://gcr.io/v2/%s/tags/list", ImageName)
	resp, err := tr.Client.Get(queryImageURL)
	if err != nil {
	return []error{fmt.Errorf("failed to list tags: %v", err)}
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
	return []error{err}
	}
	var ir imageResponse
	if err := json.Unmarshal(body, &ir); err != nil {
	return []error{err}
	}

	const deleteImageURLFmt = "https://gcr.io/v2/%s/manifests/%s"
	var errs []error
	for _, tag := range ir.Tags {
	if err := deleteDockerImageResource(tr.Client, fmt.Sprintf(deleteImageURLFmt, ImageName, tag)); err != nil {
	errs = append(errs, fmt.Errorf("failed to delete tag %s: %v", tag, err))
	}
	}

	for manifest := range ir.Manifest {
	if err := deleteDockerImageResource(tr.Client, fmt.Sprintf(deleteImageURLFmt, ImageName, manifest)); err != nil {
	errs = append(errs, fmt.Errorf("failed to delete manifest %s: %v", manifest, err))
	}
	}
	return errs
	}

	func deleteDockerImageResource(client *http.Client, url string) error {
	req, err := http.NewRequest("DELETE", url, nil)
	if err != nil {
	return fmt.Errorf("failed to get request: %v", err)
	}
	resp, err := client.Do(req)
	if err != nil {
	return fmt.Errorf("failed to delete resource: %v", err)
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted {
	return fmt.Errorf("failed to delete resource: status code = %d", resp.StatusCode)
	}
	return nil
	}

	// DeleteClusterAndImage deletes cluster and images used to create cluster.
	func (tr GKETestRunner) DeleteClusterAndImage(ctx context.Context, cfg ClusterConfig) []error {
	var errs []error
	if err := tr.StorageClient.Bucket(cfg.Bucket).Object(cfg.ImageSourceName).Delete(ctx); err != nil {
	errs = append(errs, fmt.Errorf("failed to delete storage client: %v", err))
	}
	for _, err := range tr.deleteDockerImage(ctx, cfg.ImageName) {
	errs = append(errs, fmt.Errorf("failed to delete docker image: %v", err))
	}
	if _, err := tr.ContainerService.Projects.Zones.Clusters.Delete(cfg.ProjectID, cfg.Zone, cfg.ClusterName).Context(ctx).Do(); err != nil {
	errs = append(errs, fmt.Errorf("failed to delete cluster %s: %v", cfg.ClusterName, err))
	}

	return errs
	}