cmd/go-cloud-debug-agent/debuglet.go - gocloud - Git at Google

 // Copyright 2016 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //      http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.

 // +build linux,go1.7

 package main

 import (
 	"context"
 	"encoding/json"
 	"flag"
 	"fmt"
 	"io/ioutil"
 	"log"
 	"math/rand"
 	"os"
 	"sync"
 	"time"

 	"cloud.google.com/go/cmd/go-cloud-debug-agent/internal/breakpoints"
 	debuglet "cloud.google.com/go/cmd/go-cloud-debug-agent/internal/controller"
 	"cloud.google.com/go/cmd/go-cloud-debug-agent/internal/debug"
 	"cloud.google.com/go/cmd/go-cloud-debug-agent/internal/debug/local"
 	"cloud.google.com/go/cmd/go-cloud-debug-agent/internal/valuecollector"
 	"cloud.google.com/go/compute/metadata"
 	"golang.org/x/oauth2"
 	"golang.org/x/oauth2/google"
 	cd "google.golang.org/api/clouddebugger/v2"
 )

 var (
 	appModule         = flag.String("appmodule", "", "Optional application module name.")
 	appVersion        = flag.String("appversion", "", "Optional application module version name.")
 	sourceContextFile = flag.String("sourcecontext", "", "File containing JSON-encoded source context.")
 	verbose           = flag.Bool("v", false, "Output verbose log messages.")
 	projectNumber     = flag.String("projectnumber", "", "Project number."+
 		"  If this is not set, it is read from the GCP metadata server.")
 	projectID = flag.String("projectid", "", "Project ID."+
 		"  If this is not set, it is read from the GCP metadata server.")
 	serviceAccountFile = flag.String("serviceaccountfile", "", "File containing JSON service account credentials.")
 )

 const (
 	maxCapturedStackFrames = 50
 	maxCapturedVariables   = 1000
 )

 func main() {
 	flag.Usage = usage
 	flag.Parse()
 	args := flag.Args()
 	if len(args) == 0 {
 		// The user needs to supply the name of the executable to run.
 		flag.Usage()
 		return
 	}
 	if *projectNumber == "" {
 		var err error
 		*projectNumber, err = metadata.NumericProjectID()
 		if err != nil {
 			log.Print("Debuglet initialization: ", err)
 		}
 	}
 	if *projectID == "" {
 		var err error
 		*projectID, err = metadata.ProjectID()
 		if err != nil {
 			log.Print("Debuglet initialization: ", err)
 		}
 	}
 	sourceContexts, err := readSourceContextFile(*sourceContextFile)
 	if err != nil {
 		log.Print("Reading source context file: ", err)
 	}
 	var ts oauth2.TokenSource
 	ctx := context.Background()
 	if *serviceAccountFile != "" {
 		if ts, err = serviceAcctTokenSource(ctx, *serviceAccountFile, cd.CloudDebuggerScope); err != nil {
 			log.Fatalf("Error getting credentials from file %s: %v", *serviceAccountFile, err)
 		}
 	} else if ts, err = google.DefaultTokenSource(ctx, cd.CloudDebuggerScope); err != nil {
 		log.Print("Error getting application default credentials for Cloud Debugger:", err)
 		os.Exit(103)
 	}
 	c, err := debuglet.NewController(ctx, debuglet.Options{
 		ProjectNumber:  *projectNumber,
 		ProjectID:      *projectID,
 		AppModule:      *appModule,
 		AppVersion:     *appVersion,
 		SourceContexts: sourceContexts,
 		Verbose:        *verbose,
 		TokenSource:    ts,
 	})
 	if err != nil {
 		log.Fatal("Error connecting to Cloud Debugger: ", err)
 	}
 	prog, err := local.New(args[0])
 	if err != nil {
 		log.Fatal("Error loading program: ", err)
 	}
 	// Load the program, but don't actually start it running yet.
 	if _, err = prog.Run(args[1:]...); err != nil {
 		log.Fatal("Error loading program: ", err)
 	}
 	bs := breakpoints.NewBreakpointStore(prog)

 	// Seed the random number generator.
 	rand.Seed(time.Now().UnixNano())

 	// Now we want to do two things: run the user's program, and start sending
 	// List requests periodically to the Debuglet Controller to get breakpoints
 	// to set.
 	//
 	// We want to give the Debuglet Controller a chance to give us breakpoints
 	// before we start the program, otherwise we would miss any breakpoint
 	// triggers that occur during program startup -- for example, a breakpoint on
 	// the first line of main. But if the Debuglet Controller is not responding or
 	// is returning errors, we don't want to delay starting the program
 	// indefinitely.
 	//
 	// We pass a channel to breakpointListLoop, which will close it when the first
 	// List call finishes.  Then we wait until either the channel is closed or a
 	// 5-second timer has finished before starting the program.
 	ch := make(chan bool)
 	// Start a goroutine that sends List requests to the Debuglet Controller, and
 	// sets any breakpoints it gets back.
 	go breakpointListLoop(ctx, c, bs, ch)
 	// Wait until 5 seconds have passed or breakpointListLoop has closed ch.
 	select {
 	case <-time.After(5 * time.Second):
 	case <-ch:
 	}
 	// Run the debuggee.
 	programLoop(ctx, c, bs, prog)
 }

 // usage prints a usage message to stderr and exits.
 func usage() {
 	me := "a.out"
 	if len(os.Args) >= 1 {
 		me = os.Args[0]
 	}
 	fmt.Fprintf(os.Stderr, "Usage of %s:\n", me)
 	fmt.Fprintf(os.Stderr, "\t%s [flags...] -- <program name> args...\n", me)
 	fmt.Fprintf(os.Stderr, "Flags:\n")
 	flag.PrintDefaults()
 	fmt.Fprintf(os.Stderr,
 		"See https://cloud.google.com/tools/cloud-debugger/setting-up-on-compute-engine for more information.\n")
 	os.Exit(2)
 }

 // readSourceContextFile reads a JSON-encoded source context from the given file.
 // It returns a non-empty slice on success.
 func readSourceContextFile(filename string) ([]*cd.SourceContext, error) {
 	if filename == "" {
 		return nil, nil
 	}
 	scJSON, err := ioutil.ReadFile(filename)
 	if err != nil {
 		return nil, fmt.Errorf("reading file %q: %v", filename, err)
 	}
 	var sc cd.SourceContext
 	if err = json.Unmarshal(scJSON, &sc); err != nil {
 		return nil, fmt.Errorf("parsing file %q: %v", filename, err)
 	}
 	return []*cd.SourceContext{&sc}, nil
 }

 // breakpointListLoop repeatedly calls the Debuglet Controller's List RPC, and
 // passes the results to the BreakpointStore so it can set and unset breakpoints
 // in the program.
 //
 // After the first List call finishes, ch is closed.
 func breakpointListLoop(ctx context.Context, c *debuglet.Controller, bs *breakpoints.BreakpointStore, first chan bool) {
 	const (
 		avgTimeBetweenCalls = time.Second
 		errorDelay          = 5 * time.Second
 	)

 	// randomDuration returns a random duration with expected value avg.
 	randomDuration := func(avg time.Duration) time.Duration {
 		return time.Duration(rand.Int63n(int64(2*avg + 1)))
 	}

 	var consecutiveFailures uint

 	for {
 		callStart := time.Now()
 		resp, err := c.List(ctx)
 		if err != nil && err != debuglet.ErrListUnchanged {
 			log.Printf("Debuglet controller server error: %v", err)
 		}
 		if err == nil {
 			bs.ProcessBreakpointList(resp.Breakpoints)
 		}

 		if first != nil {
 			// We've finished one call to List and set any breakpoints we received.
 			close(first)
 			first = nil
 		}

 		// Asynchronously send updates for any breakpoints that caused an error when
 		// the BreakpointStore tried to process them.  We don't wait for the update
 		// to finish before the program can exit, as we do for normal updates.
 		errorBps := bs.ErrorBreakpoints()
 		for _, bp := range errorBps {
 			go func(bp *cd.Breakpoint) {
 				if err := c.Update(ctx, bp.Id, bp); err != nil {
 					log.Printf("Failed to send breakpoint update for %s: %s", bp.Id, err)
 				}
 			}(bp)
 		}

 		// Make the next call not too soon after the one we just did.
 		delay := randomDuration(avgTimeBetweenCalls)

 		// If the call returned an error other than ErrListUnchanged, wait longer.
 		if err != nil && err != debuglet.ErrListUnchanged {
 			// Wait twice as long after each consecutive failure, to a maximum of 16x.
 			delay += randomDuration(errorDelay * (1 << consecutiveFailures))
 			if consecutiveFailures < 4 {
 				consecutiveFailures++
 			}
 		} else {
 			consecutiveFailures = 0
 		}

 		// Sleep until we reach time callStart+delay.  If we've already passed that
 		// time, time.Sleep will return immediately -- this should be the common
 		// case, since the server will delay responding to List for a while when
 		// there are no changes to report.
 		time.Sleep(callStart.Add(delay).Sub(time.Now()))
 	}
 }

 // programLoop runs the program being debugged to completion.  When a breakpoint's
 // conditions are satisfied, it sends an Update RPC to the Debuglet Controller.
 // The function returns when the program exits and all Update RPCs have finished.
 func programLoop(ctx context.Context, c *debuglet.Controller, bs *breakpoints.BreakpointStore, prog debug.Program) {
 	var wg sync.WaitGroup
 	for {
 		// Run the program until it hits a breakpoint or exits.
 		status, err := prog.Resume()
 		if err != nil {
 			break
 		}

 		// Get the breakpoints at this address whose conditions were satisfied,
 		// and remove the ones that aren't logpoints.
 		bps := bs.BreakpointsAtPC(status.PC)
 		bps = bpsWithConditionSatisfied(bps, prog)
 		for _, bp := range bps {
 			if bp.Action != "LOG" {
 				bs.RemoveBreakpoint(bp)
 			}
 		}

 		if len(bps) == 0 {
 			continue
 		}

 		// Evaluate expressions and get the stack.
 		vc := valuecollector.NewCollector(prog, maxCapturedVariables)
 		needStackFrames := false
 		for _, bp := range bps {
 			// If evaluating bp's condition didn't return an error, evaluate bp's
 			// expressions, and later get the stack frames.
 			if bp.Status == nil {
 				bp.EvaluatedExpressions = expressionValues(bp.Expressions, prog, vc)
 				needStackFrames = true
 			}
 		}
 		var (
 			stack                    []*cd.StackFrame
 			stackFramesStatusMessage *cd.StatusMessage
 		)
 		if needStackFrames {
 			stack, stackFramesStatusMessage = stackFrames(prog, vc)
 		}

 		// Read variable values from the program.
 		variableTable := vc.ReadValues()

 		// Start a goroutine to send updates to the Debuglet Controller or write
 		// to logs, concurrently with resuming the program.
 		// TODO: retry Update on failure.
 		for _, bp := range bps {
 			wg.Add(1)
 			switch bp.Action {
 			case "LOG":
 				go func(format string, evaluatedExpressions []*cd.Variable) {
 					s := valuecollector.LogString(format, evaluatedExpressions, variableTable)
 					log.Print(s)
 					wg.Done()
 				}(bp.LogMessageFormat, bp.EvaluatedExpressions)
 				bp.Status = nil
 				bp.EvaluatedExpressions = nil
 			default:
 				go func(bp *cd.Breakpoint) {
 					defer wg.Done()
 					bp.IsFinalState = true
 					if bp.Status == nil {
 						// If evaluating bp's condition didn't return an error, include the
 						// stack frames, variable table, and any status message produced when
 						// getting the stack frames.
 						bp.StackFrames = stack
 						bp.VariableTable = variableTable
 						bp.Status = stackFramesStatusMessage
 					}
 					if err := c.Update(ctx, bp.Id, bp); err != nil {
 						log.Printf("Failed to send breakpoint update for %s: %s", bp.Id, err)
 					}
 				}(bp)
 			}
 		}
 	}

 	// Wait for all updates to finish before returning.
 	wg.Wait()
 }

 // bpsWithConditionSatisfied returns the breakpoints whose conditions are true
 // (or that do not have a condition.)
 func bpsWithConditionSatisfied(bpsIn []*cd.Breakpoint, prog debug.Program) []*cd.Breakpoint {
 	var bpsOut []*cd.Breakpoint
 	for _, bp := range bpsIn {
 		cond, err := condTruth(bp.Condition, prog)
 		if err != nil {
 			bp.Status = errorStatusMessage(err.Error(), refersToBreakpointCondition)
 			// Include bp in the list to be updated when there's an error, so that
 			// the user gets a response.
 			bpsOut = append(bpsOut, bp)
 		} else if cond {
 			bpsOut = append(bpsOut, bp)
 		}
 	}
 	return bpsOut
 }

 // condTruth evaluates a condition.
 func condTruth(condition string, prog debug.Program) (bool, error) {
 	if condition == "" {
 		// A condition wasn't set.
 		return true, nil
 	}
 	val, err := prog.Evaluate(condition)
 	if err != nil {
 		return false, err
 	}
 	if v, ok := val.(bool); !ok {
 		return false, fmt.Errorf("condition expression has type %T, should be bool", val)
 	} else {
 		return v, nil
 	}
 }

 // expressionValues evaluates a slice of expressions and returns a []*cd.Variable
 // containing the results.
 // If the result of an expression evaluation refers to values from the program's
 // memory (e.g., the expression evaluates to a slice) a corresponding variable is
 // added to the value collector, to be read later.
 func expressionValues(expressions []string, prog debug.Program, vc *valuecollector.Collector) []*cd.Variable {
 	evaluatedExpressions := make([]*cd.Variable, len(expressions))
 	for i, exp := range expressions {
 		ee := &cd.Variable{Name: exp}
 		evaluatedExpressions[i] = ee
 		if val, err := prog.Evaluate(exp); err != nil {
 			ee.Status = errorStatusMessage(err.Error(), refersToBreakpointExpression)
 		} else {
 			vc.FillValue(val, ee)
 		}
 	}
 	return evaluatedExpressions
 }

 // stackFrames returns a stack trace for the program.  It passes references to
 // function parameters and local variables to the value collector, so it can read
 // their values later.
 func stackFrames(prog debug.Program, vc *valuecollector.Collector) ([]*cd.StackFrame, *cd.StatusMessage) {
 	frames, err := prog.Frames(maxCapturedStackFrames)
 	if err != nil {
 		return nil, errorStatusMessage("Error getting stack: "+err.Error(), refersToUnspecified)
 	}
 	stackFrames := make([]*cd.StackFrame, len(frames))
 	for i, f := range frames {
 		frame := &cd.StackFrame{}
 		frame.Function = f.Function
 		for _, v := range f.Params {
 			frame.Arguments = append(frame.Arguments, vc.AddVariable(debug.LocalVar(v)))
 		}
 		for _, v := range f.Vars {
 			frame.Locals = append(frame.Locals, vc.AddVariable(v))
 		}
 		frame.Location = &cd.SourceLocation{
 			Path: f.File,
 			Line: int64(f.Line),
 		}
 		stackFrames[i] = frame
 	}
 	return stackFrames, nil
 }

 // errorStatusMessage returns a *cd.StatusMessage indicating an error,
 // with the given message and refersTo field.
 func errorStatusMessage(msg string, refersTo int) *cd.StatusMessage {
 	return &cd.StatusMessage{
 		Description: &cd.FormatMessage{Format: "$0", Parameters: []string{msg}},
 		IsError:     true,
 		RefersTo:    refersToString[refersTo],
 	}
 }

 const (
 	// RefersTo values for cd.StatusMessage.
 	refersToUnspecified = iota
 	refersToBreakpointCondition
 	refersToBreakpointExpression
 )

 // refersToString contains the strings for each refersTo value.
 // See the definition of StatusMessage in the v2/clouddebugger package.
 var refersToString = map[int]string{
 	refersToUnspecified:          "UNSPECIFIED",
 	refersToBreakpointCondition:  "BREAKPOINT_CONDITION",
 	refersToBreakpointExpression: "BREAKPOINT_EXPRESSION",
 }

 func serviceAcctTokenSource(ctx context.Context, filename string, scope ...string) (oauth2.TokenSource, error) {
 	data, err := ioutil.ReadFile(filename)
 	if err != nil {
 		return nil, fmt.Errorf("cannot read service account file: %v", err)
 	}
 	cfg, err := google.JWTConfigFromJSON(data, scope...)
 	if err != nil {
 		return nil, fmt.Errorf("google.JWTConfigFromJSON: %v", err)
 	}
 	return cfg.TokenSource(ctx), nil
 }
	// Copyright 2016 Google LLC
	//
	// Licensed under the Apache License, Version 2.0 (the "License");
	// you may not use this file except in compliance with the License.
	// You may obtain a copy of the License at
	//
	// http://www.apache.org/licenses/LICENSE-2.0
	//
	// Unless required by applicable law or agreed to in writing, software
	// distributed under the License is distributed on an "AS IS" BASIS,
	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	// See the License for the specific language governing permissions and
	// limitations under the License.

	// +build linux,go1.7

	package main

	import (
	"context"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"math/rand"
	"os"
	"sync"
	"time"

	"cloud.google.com/go/cmd/go-cloud-debug-agent/internal/breakpoints"
	debuglet "cloud.google.com/go/cmd/go-cloud-debug-agent/internal/controller"
	"cloud.google.com/go/cmd/go-cloud-debug-agent/internal/debug"
	"cloud.google.com/go/cmd/go-cloud-debug-agent/internal/debug/local"
	"cloud.google.com/go/cmd/go-cloud-debug-agent/internal/valuecollector"
	"cloud.google.com/go/compute/metadata"
	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"
	cd "google.golang.org/api/clouddebugger/v2"
	)

	var (
	appModule = flag.String("appmodule", "", "Optional application module name.")
	appVersion = flag.String("appversion", "", "Optional application module version name.")
	sourceContextFile = flag.String("sourcecontext", "", "File containing JSON-encoded source context.")
	verbose = flag.Bool("v", false, "Output verbose log messages.")
	projectNumber = flag.String("projectnumber", "", "Project number."+
	" If this is not set, it is read from the GCP metadata server.")
	projectID = flag.String("projectid", "", "Project ID."+
	" If this is not set, it is read from the GCP metadata server.")
	serviceAccountFile = flag.String("serviceaccountfile", "", "File containing JSON service account credentials.")
	)

	const (
	maxCapturedStackFrames = 50
	maxCapturedVariables = 1000
	)

	func main() {
	flag.Usage = usage
	flag.Parse()
	args := flag.Args()
	if len(args) == 0 {
	// The user needs to supply the name of the executable to run.
	flag.Usage()
	return
	}
	if *projectNumber == "" {
	var err error
	*projectNumber, err = metadata.NumericProjectID()
	if err != nil {
	log.Print("Debuglet initialization: ", err)
	}
	}
	if *projectID == "" {
	var err error
	*projectID, err = metadata.ProjectID()
	if err != nil {
	log.Print("Debuglet initialization: ", err)
	}
	}
	sourceContexts, err := readSourceContextFile(*sourceContextFile)
	if err != nil {
	log.Print("Reading source context file: ", err)
	}
	var ts oauth2.TokenSource
	ctx := context.Background()
	if *serviceAccountFile != "" {
	if ts, err = serviceAcctTokenSource(ctx, *serviceAccountFile, cd.CloudDebuggerScope); err != nil {
	log.Fatalf("Error getting credentials from file %s: %v", *serviceAccountFile, err)
	}
	} else if ts, err = google.DefaultTokenSource(ctx, cd.CloudDebuggerScope); err != nil {
	log.Print("Error getting application default credentials for Cloud Debugger:", err)
	os.Exit(103)
	}
	c, err := debuglet.NewController(ctx, debuglet.Options{
	ProjectNumber: *projectNumber,
	ProjectID: *projectID,
	AppModule: *appModule,
	AppVersion: *appVersion,
	SourceContexts: sourceContexts,
	Verbose: *verbose,
	TokenSource: ts,
	})
	if err != nil {
	log.Fatal("Error connecting to Cloud Debugger: ", err)
	}
	prog, err := local.New(args[0])
	if err != nil {
	log.Fatal("Error loading program: ", err)
	}
	// Load the program, but don't actually start it running yet.
	if _, err = prog.Run(args[1:]...); err != nil {
	log.Fatal("Error loading program: ", err)
	}
	bs := breakpoints.NewBreakpointStore(prog)

	// Seed the random number generator.
	rand.Seed(time.Now().UnixNano())

	// Now we want to do two things: run the user's program, and start sending
	// List requests periodically to the Debuglet Controller to get breakpoints
	// to set.
	//
	// We want to give the Debuglet Controller a chance to give us breakpoints
	// before we start the program, otherwise we would miss any breakpoint
	// triggers that occur during program startup -- for example, a breakpoint on
	// the first line of main. But if the Debuglet Controller is not responding or
	// is returning errors, we don't want to delay starting the program
	// indefinitely.
	//
	// We pass a channel to breakpointListLoop, which will close it when the first
	// List call finishes. Then we wait until either the channel is closed or a
	// 5-second timer has finished before starting the program.
	ch := make(chan bool)
	// Start a goroutine that sends List requests to the Debuglet Controller, and
	// sets any breakpoints it gets back.
	go breakpointListLoop(ctx, c, bs, ch)
	// Wait until 5 seconds have passed or breakpointListLoop has closed ch.
	select {
	case <-time.After(5 * time.Second):
	case <-ch:
	}
	// Run the debuggee.
	programLoop(ctx, c, bs, prog)
	}

	// usage prints a usage message to stderr and exits.
	func usage() {
	me := "a.out"
	if len(os.Args) >= 1 {
	me = os.Args[0]
	}
	fmt.Fprintf(os.Stderr, "Usage of %s:\n", me)
	fmt.Fprintf(os.Stderr, "\t%s [flags...] -- <program name> args...\n", me)
	fmt.Fprintf(os.Stderr, "Flags:\n")
	flag.PrintDefaults()
	fmt.Fprintf(os.Stderr,
	"See https://cloud.google.com/tools/cloud-debugger/setting-up-on-compute-engine for more information.\n")
	os.Exit(2)
	}

	// readSourceContextFile reads a JSON-encoded source context from the given file.
	// It returns a non-empty slice on success.
	func readSourceContextFile(filename string) ([]*cd.SourceContext, error) {
	if filename == "" {
	return nil, nil
	}
	scJSON, err := ioutil.ReadFile(filename)
	if err != nil {
	return nil, fmt.Errorf("reading file %q: %v", filename, err)
	}
	var sc cd.SourceContext
	if err = json.Unmarshal(scJSON, &sc); err != nil {
	return nil, fmt.Errorf("parsing file %q: %v", filename, err)
	}
	return []*cd.SourceContext{&sc}, nil
	}

	// breakpointListLoop repeatedly calls the Debuglet Controller's List RPC, and
	// passes the results to the BreakpointStore so it can set and unset breakpoints
	// in the program.
	//
	// After the first List call finishes, ch is closed.
	func breakpointListLoop(ctx context.Context, c debuglet.Controller, bs breakpoints.BreakpointStore, first chan bool) {
	const (
	avgTimeBetweenCalls = time.Second
	errorDelay = 5 * time.Second
	)

	// randomDuration returns a random duration with expected value avg.
	randomDuration := func(avg time.Duration) time.Duration {
	return time.Duration(rand.Int63n(int64(2*avg + 1)))
	}

	var consecutiveFailures uint

	for {
	callStart := time.Now()
	resp, err := c.List(ctx)
	if err != nil && err != debuglet.ErrListUnchanged {
	log.Printf("Debuglet controller server error: %v", err)
	}
	if err == nil {
	bs.ProcessBreakpointList(resp.Breakpoints)
	}

	if first != nil {
	// We've finished one call to List and set any breakpoints we received.
	close(first)
	first = nil
	}

	// Asynchronously send updates for any breakpoints that caused an error when
	// the BreakpointStore tried to process them. We don't wait for the update
	// to finish before the program can exit, as we do for normal updates.
	errorBps := bs.ErrorBreakpoints()
	for _, bp := range errorBps {
	go func(bp *cd.Breakpoint) {
	if err := c.Update(ctx, bp.Id, bp); err != nil {
	log.Printf("Failed to send breakpoint update for %s: %s", bp.Id, err)
	}
	}(bp)
	}

	// Make the next call not too soon after the one we just did.
	delay := randomDuration(avgTimeBetweenCalls)

	// If the call returned an error other than ErrListUnchanged, wait longer.
	if err != nil && err != debuglet.ErrListUnchanged {
	// Wait twice as long after each consecutive failure, to a maximum of 16x.
	delay += randomDuration(errorDelay * (1 << consecutiveFailures))
	if consecutiveFailures < 4 {
	consecutiveFailures++
	}
	} else {
	consecutiveFailures = 0
	}

	// Sleep until we reach time callStart+delay. If we've already passed that
	// time, time.Sleep will return immediately -- this should be the common
	// case, since the server will delay responding to List for a while when
	// there are no changes to report.
	time.Sleep(callStart.Add(delay).Sub(time.Now()))
	}
	}

	// programLoop runs the program being debugged to completion. When a breakpoint's
	// conditions are satisfied, it sends an Update RPC to the Debuglet Controller.
	// The function returns when the program exits and all Update RPCs have finished.
	func programLoop(ctx context.Context, c debuglet.Controller, bs breakpoints.BreakpointStore, prog debug.Program) {
	var wg sync.WaitGroup
	for {
	// Run the program until it hits a breakpoint or exits.
	status, err := prog.Resume()
	if err != nil {
	break
	}

	// Get the breakpoints at this address whose conditions were satisfied,
	// and remove the ones that aren't logpoints.
	bps := bs.BreakpointsAtPC(status.PC)
	bps = bpsWithConditionSatisfied(bps, prog)
	for _, bp := range bps {
	if bp.Action != "LOG" {
	bs.RemoveBreakpoint(bp)
	}
	}

	if len(bps) == 0 {
	continue
	}

	// Evaluate expressions and get the stack.
	vc := valuecollector.NewCollector(prog, maxCapturedVariables)
	needStackFrames := false
	for _, bp := range bps {
	// If evaluating bp's condition didn't return an error, evaluate bp's
	// expressions, and later get the stack frames.
	if bp.Status == nil {
	bp.EvaluatedExpressions = expressionValues(bp.Expressions, prog, vc)
	needStackFrames = true
	}
	}
	var (
	stack []*cd.StackFrame
	stackFramesStatusMessage *cd.StatusMessage
	)
	if needStackFrames {
	stack, stackFramesStatusMessage = stackFrames(prog, vc)
	}

	// Read variable values from the program.
	variableTable := vc.ReadValues()

	// Start a goroutine to send updates to the Debuglet Controller or write
	// to logs, concurrently with resuming the program.
	// TODO: retry Update on failure.
	for _, bp := range bps {
	wg.Add(1)
	switch bp.Action {
	case "LOG":
	go func(format string, evaluatedExpressions []*cd.Variable) {
	s := valuecollector.LogString(format, evaluatedExpressions, variableTable)
	log.Print(s)
	wg.Done()
	}(bp.LogMessageFormat, bp.EvaluatedExpressions)
	bp.Status = nil
	bp.EvaluatedExpressions = nil
	default:
	go func(bp *cd.Breakpoint) {
	defer wg.Done()
	bp.IsFinalState = true
	if bp.Status == nil {
	// If evaluating bp's condition didn't return an error, include the
	// stack frames, variable table, and any status message produced when
	// getting the stack frames.
	bp.StackFrames = stack
	bp.VariableTable = variableTable
	bp.Status = stackFramesStatusMessage
	}
	if err := c.Update(ctx, bp.Id, bp); err != nil {
	log.Printf("Failed to send breakpoint update for %s: %s", bp.Id, err)
	}
	}(bp)
	}
	}
	}

	// Wait for all updates to finish before returning.
	wg.Wait()
	}

	// bpsWithConditionSatisfied returns the breakpoints whose conditions are true
	// (or that do not have a condition.)
	func bpsWithConditionSatisfied(bpsIn []cd.Breakpoint, prog debug.Program) []cd.Breakpoint {
	var bpsOut []*cd.Breakpoint
	for _, bp := range bpsIn {
	cond, err := condTruth(bp.Condition, prog)
	if err != nil {
	bp.Status = errorStatusMessage(err.Error(), refersToBreakpointCondition)
	// Include bp in the list to be updated when there's an error, so that
	// the user gets a response.
	bpsOut = append(bpsOut, bp)
	} else if cond {
	bpsOut = append(bpsOut, bp)
	}
	}
	return bpsOut
	}

	// condTruth evaluates a condition.
	func condTruth(condition string, prog debug.Program) (bool, error) {
	if condition == "" {
	// A condition wasn't set.
	return true, nil
	}
	val, err := prog.Evaluate(condition)
	if err != nil {
	return false, err
	}
	if v, ok := val.(bool); !ok {
	return false, fmt.Errorf("condition expression has type %T, should be bool", val)
	} else {
	return v, nil
	}
	}

	// expressionValues evaluates a slice of expressions and returns a []*cd.Variable
	// containing the results.
	// If the result of an expression evaluation refers to values from the program's
	// memory (e.g., the expression evaluates to a slice) a corresponding variable is
	// added to the value collector, to be read later.
	func expressionValues(expressions []string, prog debug.Program, vc valuecollector.Collector) []cd.Variable {
	evaluatedExpressions := make([]*cd.Variable, len(expressions))
	for i, exp := range expressions {
	ee := &cd.Variable{Name: exp}
	evaluatedExpressions[i] = ee
	if val, err := prog.Evaluate(exp); err != nil {
	ee.Status = errorStatusMessage(err.Error(), refersToBreakpointExpression)
	} else {
	vc.FillValue(val, ee)
	}
	}
	return evaluatedExpressions
	}

	// stackFrames returns a stack trace for the program. It passes references to
	// function parameters and local variables to the value collector, so it can read
	// their values later.
	func stackFrames(prog debug.Program, vc valuecollector.Collector) ([]cd.StackFrame, *cd.StatusMessage) {
	frames, err := prog.Frames(maxCapturedStackFrames)
	if err != nil {
	return nil, errorStatusMessage("Error getting stack: "+err.Error(), refersToUnspecified)
	}
	stackFrames := make([]*cd.StackFrame, len(frames))
	for i, f := range frames {
	frame := &cd.StackFrame{}
	frame.Function = f.Function
	for _, v := range f.Params {
	frame.Arguments = append(frame.Arguments, vc.AddVariable(debug.LocalVar(v)))
	}
	for _, v := range f.Vars {
	frame.Locals = append(frame.Locals, vc.AddVariable(v))
	}
	frame.Location = &cd.SourceLocation{
	Path: f.File,
	Line: int64(f.Line),
	}
	stackFrames[i] = frame
	}
	return stackFrames, nil
	}

	// errorStatusMessage returns a *cd.StatusMessage indicating an error,
	// with the given message and refersTo field.
	func errorStatusMessage(msg string, refersTo int) *cd.StatusMessage {
	return &cd.StatusMessage{
	Description: &cd.FormatMessage{Format: "$0", Parameters: []string{msg}},
	IsError: true,
	RefersTo: refersToString[refersTo],
	}
	}

	const (
	// RefersTo values for cd.StatusMessage.
	refersToUnspecified = iota
	refersToBreakpointCondition
	refersToBreakpointExpression
	)

	// refersToString contains the strings for each refersTo value.
	// See the definition of StatusMessage in the v2/clouddebugger package.
	var refersToString = map[int]string{
	refersToUnspecified: "UNSPECIFIED",
	refersToBreakpointCondition: "BREAKPOINT_CONDITION",
	refersToBreakpointExpression: "BREAKPOINT_EXPRESSION",
	}

	func serviceAcctTokenSource(ctx context.Context, filename string, scope ...string) (oauth2.TokenSource, error) {
	data, err := ioutil.ReadFile(filename)
	if err != nil {
	return nil, fmt.Errorf("cannot read service account file: %v", err)
	}
	cfg, err := google.JWTConfigFromJSON(data, scope...)
	if err != nil {
	return nil, fmt.Errorf("google.JWTConfigFromJSON: %v", err)
	}
	return cfg.TokenSource(ctx), nil
	}