feature(bigquery): improve error guidance for streaming inserts (#3024)
* feature(bigquery): improve error guidance for streaming inserts
This PR attempts to clarify guidance for users of the streaming insert mechanism.
BigQuery enforces a variety of limitations on client requests (size, qps, etc). When
those limits are more severely violated, BigQuery replies with less specific information
about the issue, and instead generates an unstructured HTTP 400 client error response.
This PR adds guidance to doc.go about this behavior, and adds an integration test that
induces these errors to ensure the error surface is behaving as documented.
diff --git a/bigquery/doc.go b/bigquery/doc.go
index e540793..3a70714 100644
--- a/bigquery/doc.go
+++ b/bigquery/doc.go
@@ -253,9 +253,9 @@
// Poll the job for completion if desired, as above.
To upload, first define a type that implements the ValueSaver interface, which has a single method named Save.
-Then create an Uploader, and call its Put method with a slice of values.
+Then create an Inserter, and call its Put method with a slice of values.
- u := table.Uploader()
+ u := table.Inserter()
// Item implements the ValueSaver interface.
items := []*Item{
{Name: "n1", Size: 32.6, Count: 7},
@@ -285,6 +285,9 @@
// TODO: Handle error.
}
+BigQuery allows for higher throughput when omitting insertion IDs. To enable this,
+specify the sentinel `NoDedupeID` value for the insertion ID when implementing a ValueSaver.
+
Extracting
If you've been following so far, extracting data from a BigQuery table
@@ -298,11 +301,16 @@
Errors
-Errors returned by this client are often of the type `[googleapi.Error](https://godoc.org/google.golang.org/api/googleapi#Error)`.
-These errors can be introspected for more information by type asserting to the richer `googleapi.Error` type. For example:
+Errors returned by this client are often of the type googleapi.Error: https://godoc.org/google.golang.org/api/googleapi#Error
+
+These errors can be introspected for more information by type asserting to the richer *googleapi.Error type. For example:
if e, ok := err.(*googleapi.Error); ok {
if e.Code = 409 { ... }
- }
+ }
+
+In some cases, your client may received unstructured googleapi.Error error responses. In such cases, it is likely that
+you have exceeded BigQuery request limits, documented at: https://cloud.google.com/bigquery/quotas
+
*/
package bigquery // import "cloud.google.com/go/bigquery"
diff --git a/bigquery/integration_test.go b/bigquery/integration_test.go
index 7367e72..58742dc 100644
--- a/bigquery/integration_test.go
+++ b/bigquery/integration_test.go
@@ -1062,6 +1062,69 @@
it, [][]Value{{int64(10)}})
}
+func TestIntegration_InsertErrors(t *testing.T) {
+ if client == nil {
+ t.Skip("Integration tests skipped")
+ }
+ ctx := context.Background()
+ table := newTable(t, schema)
+ defer table.Delete(ctx)
+
+ ins := table.Inserter()
+ var saverRows []*ValuesSaver
+
+ // badSaver represents an excessively sized (>5Mb) row message for insertion.
+ badSaver := &ValuesSaver{
+ Schema: schema,
+ InsertID: NoDedupeID,
+ Row: []Value{strings.Repeat("X", 5242881), []Value{int64(1)}, []Value{true}},
+ }
+
+ // Case 1: A single oversized row.
+ saverRows = append(saverRows, badSaver)
+ err := ins.Put(ctx, saverRows)
+ if err == nil {
+ t.Errorf("Wanted row size error, got successful insert.")
+ }
+ if _, ok := err.(PutMultiError); !ok {
+ t.Errorf("Wanted PutMultiError, but wasn't: %v", err)
+ }
+ got := putError(err)
+ want := "Maximum allowed row size exceeded"
+ if !strings.Contains(got, want) {
+ t.Errorf("Error didn't contain expected substring (%s): %s", want, got)
+ }
+ // Case 2: The overall request size > 10MB)
+ // 2x 5MB rows
+ saverRows = append(saverRows, badSaver)
+ err = ins.Put(ctx, saverRows)
+ if err == nil {
+ t.Errorf("Wanted structured size error, got successful insert.")
+ }
+ e, ok := err.(*googleapi.Error)
+ if !ok {
+ t.Errorf("Wanted googleapi.Error, got: %v", err)
+ }
+ want = "Request payload size exceeds the limit"
+ if !strings.Contains(e.Message, want) {
+ t.Errorf("Error didn't contain expected message (%s): %s", want, e.Message)
+ }
+ // Case 3: Very Large Request
+ // Request so large it gets rejected by an intermediate (4x 5MB rows)
+ saverRows = append(saverRows, saverRows...)
+ err = ins.Put(ctx, saverRows)
+ if err == nil {
+ t.Errorf("Wanted error, got successful insert.")
+ }
+ e, ok = err.(*googleapi.Error)
+ if !ok {
+ t.Errorf("wanted googleapi.Error, got: %v", err)
+ }
+ if e.Code != http.StatusBadRequest {
+ t.Errorf("Wanted HTTP 400, got %d", e.Code)
+ }
+}
+
func TestIntegration_InsertAndRead(t *testing.T) {
if client == nil {
t.Skip("Integration tests skipped")
