bigquery: allow opt-out on streaming deduplication
This change adds a new sentinel NoInsertID value for
indicating that a Saver should not leverage best-effort
deduplication when using an Inserter for streaming data
into a table.
Currently, any Saver that specifies an empty insertion ID
gets a randomly generated insertID to facilitate deduplication
and retry scenarios. Specifying the NoInsertID sentinel
omits the use of insertIDs in the underlying request.
Fixes: #1618
Change-Id: I30638a6a40323973c146f2ef5e39e076ee07db02
Reviewed-on: https://code-review.googlesource.com/c/gocloud/+/47570
Reviewed-by: kokoro <noreply+kokoro@google.com>
Reviewed-by: Jean de Klerk <deklerk@google.com>
diff --git a/bigquery/inserter.go b/bigquery/inserter.go
index 2a12bb3..789cd00 100644
--- a/bigquery/inserter.go
+++ b/bigquery/inserter.go
@@ -24,6 +24,11 @@
bq "google.golang.org/api/bigquery/v2"
)
+// NoDedupeID indicates a streaming insert row wants to opt out of best-effort
+// deduplication.
+// It is EXPERIMENTAL and subject to change or removal without notice.
+const NoDedupeID = "NoDedupeID"
+
// An Inserter does streaming inserts into a BigQuery table.
// It is safe for concurrent use.
type Inserter struct {
@@ -200,7 +205,9 @@
if err != nil {
return nil, err
}
- if insertID == "" {
+ if insertID == NoDedupeID {
+ insertID = ""
+ } else if insertID == "" {
insertID = randomIDFn()
}
m := make(map[string]bq.JsonValue)
diff --git a/bigquery/inserter_test.go b/bigquery/inserter_test.go
index 0fafeac..8216ecb 100644
--- a/bigquery/inserter_test.go
+++ b/bigquery/inserter_test.go
@@ -56,11 +56,13 @@
savers: []ValueSaver{
testSaver{row: map[string]Value{"one": 1}},
testSaver{row: map[string]Value{"two": 2}},
+ testSaver{insertID: NoDedupeID, row: map[string]Value{"three": 3}},
},
req: &bq.TableDataInsertAllRequest{
Rows: []*bq.TableDataInsertAllRequestRows{
{InsertId: "1", Json: map[string]bq.JsonValue{"one": 1}},
{InsertId: "2", Json: map[string]bq.JsonValue{"two": 2}},
+ {InsertId: "", Json: map[string]bq.JsonValue{"three": 3}},
},
},
},
