blob: 99d299da32016f73f0d06b00ccaad9d0db15e0a0 [file] [log] [blame]
// Copyright 2015 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package bigquery
import (
"context"
"errors"
"fmt"
"testing"
"cloud.google.com/go/internal/testutil"
"github.com/google/go-cmp/cmp"
bq "google.golang.org/api/bigquery/v2"
"google.golang.org/api/iterator"
)
type fetchResponse struct {
result *fetchPageResult // The result to return.
err error // The error to return.
}
// pageFetcherStub services fetch requests by returning data from an in-memory list of values.
type pageFetcherStub struct {
fetchResponses map[string]fetchResponse
err error
}
func (pf *pageFetcherStub) fetchPage(ctx context.Context, _ *rowSource, _ Schema, _ uint64, _ int64, pageToken string) (*fetchPageResult, error) {
call, ok := pf.fetchResponses[pageToken]
if !ok {
pf.err = fmt.Errorf("Unexpected page token: %q", pageToken)
}
return call.result, call.err
}
func TestRowIteratorCacheBehavior(t *testing.T) {
testSchema := &bq.TableSchema{
Fields: []*bq.TableFieldSchema{
{Type: "INTEGER", Name: "field1"},
{Type: "STRING", Name: "field2"},
},
}
testRows := []*bq.TableRow{
{F: []*bq.TableCell{
{V: "1"},
{V: "foo"},
},
},
}
convertedSchema := bqToSchema(testSchema)
convertedRows, _ := convertRows(testRows, convertedSchema)
testCases := []struct {
inSource *rowSource
inSchema Schema
inStartIndex uint64
inPageSize int64
inPageToken string
wantErr error
wantResult *fetchPageResult
}{
{
inSource: &rowSource{},
wantErr: errNoCacheData,
},
{
// primary success case: schema in cache
inSource: &rowSource{
cachedSchema: testSchema,
cachedRows: testRows,
},
wantResult: &fetchPageResult{
totalRows: uint64(len(convertedRows)),
schema: convertedSchema,
rows: convertedRows,
},
},
{
// secondary success case: schema provided
inSource: &rowSource{
cachedRows: testRows,
cachedNextToken: "foo",
},
inSchema: convertedSchema,
wantResult: &fetchPageResult{
totalRows: uint64(len(convertedRows)),
schema: convertedSchema,
rows: convertedRows,
pageToken: "foo",
},
},
{
// misaligned page size.
inSource: &rowSource{
cachedSchema: testSchema,
cachedRows: testRows,
},
inPageSize: 99,
wantErr: errNoCacheData,
},
{
// nonzero start.
inSource: &rowSource{
cachedSchema: testSchema,
cachedRows: testRows,
},
inStartIndex: 1,
wantErr: errNoCacheData,
},
{
// data without schema
inSource: &rowSource{
cachedSchema: testSchema,
cachedRows: testRows,
},
inStartIndex: 1,
wantErr: errNoCacheData,
},
{
// data without schema
inSource: &rowSource{
cachedSchema: testSchema,
cachedRows: testRows,
},
inStartIndex: 1,
wantErr: errNoCacheData,
},
}
for _, tc := range testCases {
gotResp, gotErr := fetchCachedPage(context.Background(), tc.inSource, tc.inSchema, tc.inStartIndex, tc.inPageSize, tc.inPageToken)
if gotErr != tc.wantErr {
t.Errorf("err mismatch. got %v, want %v", gotErr, tc.wantErr)
} else {
if diff := testutil.Diff(gotResp, tc.wantResult,
cmp.AllowUnexported(fetchPageResult{}, rowSource{}, Job{}, Client{}, Table{})); diff != "" {
t.Errorf("response diff (got=-, want=+):\n%s", diff)
}
}
}
}
func TestIterator(t *testing.T) {
var (
iiSchema = Schema{
{Type: IntegerFieldType},
{Type: IntegerFieldType},
}
siSchema = Schema{
{Type: StringFieldType},
{Type: IntegerFieldType},
}
)
fetchFailure := errors.New("fetch failure")
testCases := []struct {
desc string
pageToken string
fetchResponses map[string]fetchResponse
want [][]Value
wantErr error
wantSchema Schema
wantTotalRows uint64
}{
{
desc: "Iteration over single empty page",
fetchResponses: map[string]fetchResponse{
"": {
result: &fetchPageResult{
pageToken: "",
rows: [][]Value{},
schema: Schema{},
},
},
},
want: [][]Value{},
wantSchema: Schema{},
},
{
desc: "Iteration over single page",
fetchResponses: map[string]fetchResponse{
"": {
result: &fetchPageResult{
pageToken: "",
rows: [][]Value{{1, 2}, {11, 12}},
schema: iiSchema,
totalRows: 4,
},
},
},
want: [][]Value{{1, 2}, {11, 12}},
wantSchema: iiSchema,
wantTotalRows: 4,
},
{
desc: "Iteration over single page with different schema",
fetchResponses: map[string]fetchResponse{
"": {
result: &fetchPageResult{
pageToken: "",
rows: [][]Value{{"1", 2}, {"11", 12}},
schema: siSchema,
},
},
},
want: [][]Value{{"1", 2}, {"11", 12}},
wantSchema: siSchema,
},
{
desc: "Iteration over two pages",
fetchResponses: map[string]fetchResponse{
"": {
result: &fetchPageResult{
pageToken: "a",
rows: [][]Value{{1, 2}, {11, 12}},
schema: iiSchema,
totalRows: 4,
},
},
"a": {
result: &fetchPageResult{
pageToken: "",
rows: [][]Value{{101, 102}, {111, 112}},
schema: iiSchema,
totalRows: 4,
},
},
},
want: [][]Value{{1, 2}, {11, 12}, {101, 102}, {111, 112}},
wantSchema: iiSchema,
wantTotalRows: 4,
},
{
desc: "Server response includes empty page",
fetchResponses: map[string]fetchResponse{
"": {
result: &fetchPageResult{
pageToken: "a",
rows: [][]Value{{1, 2}, {11, 12}},
schema: iiSchema,
},
},
"a": {
result: &fetchPageResult{
pageToken: "b",
rows: [][]Value{},
schema: iiSchema,
},
},
"b": {
result: &fetchPageResult{
pageToken: "",
rows: [][]Value{{101, 102}, {111, 112}},
schema: iiSchema,
},
},
},
want: [][]Value{{1, 2}, {11, 12}, {101, 102}, {111, 112}},
wantSchema: iiSchema,
},
{
desc: "Fetch error",
fetchResponses: map[string]fetchResponse{
"": {
result: &fetchPageResult{
pageToken: "a",
rows: [][]Value{{1, 2}, {11, 12}},
schema: iiSchema,
},
},
"a": {
// We returns some data from this fetch, but also an error.
// So the end result should include only data from the previous fetch.
err: fetchFailure,
result: &fetchPageResult{
pageToken: "b",
rows: [][]Value{{101, 102}, {111, 112}},
schema: iiSchema,
},
},
},
want: [][]Value{{1, 2}, {11, 12}},
wantErr: fetchFailure,
wantSchema: iiSchema,
},
{
desc: "Skip over an entire page",
pageToken: "a",
fetchResponses: map[string]fetchResponse{
"": {
result: &fetchPageResult{
pageToken: "a",
rows: [][]Value{{1, 2}, {11, 12}},
schema: iiSchema,
},
},
"a": {
result: &fetchPageResult{
pageToken: "",
rows: [][]Value{{101, 102}, {111, 112}},
schema: iiSchema,
},
},
},
want: [][]Value{{101, 102}, {111, 112}},
wantSchema: iiSchema,
},
{
desc: "Skip beyond all data",
pageToken: "b",
fetchResponses: map[string]fetchResponse{
"": {
result: &fetchPageResult{
pageToken: "a",
rows: [][]Value{{1, 2}, {11, 12}},
schema: iiSchema,
},
},
"a": {
result: &fetchPageResult{
pageToken: "b",
rows: [][]Value{{101, 102}, {111, 112}},
schema: iiSchema,
},
},
"b": {
result: &fetchPageResult{},
},
},
// In this test case, Next will return false on its first call,
// so we won't even attempt to call Get.
want: [][]Value{},
wantSchema: Schema{},
},
}
for _, tc := range testCases {
pf := &pageFetcherStub{
fetchResponses: tc.fetchResponses,
}
it := newRowIterator(context.Background(), nil, pf.fetchPage)
it.PageInfo().Token = tc.pageToken
values, schema, totalRows, err := consumeRowIterator(it)
if err != tc.wantErr {
t.Fatalf("%s: got %v, want %v", tc.desc, err, tc.wantErr)
}
if (len(values) != 0 || len(tc.want) != 0) && !testutil.Equal(values, tc.want) {
t.Errorf("%s: values:\ngot: %v\nwant:%v", tc.desc, values, tc.want)
}
if (len(schema) != 0 || len(tc.wantSchema) != 0) && !testutil.Equal(schema, tc.wantSchema) {
t.Errorf("%s: iterator.Schema:\ngot: %v\nwant: %v", tc.desc, schema, tc.wantSchema)
}
if totalRows != tc.wantTotalRows {
t.Errorf("%s: totalRows: got %d, want %d", tc.desc, totalRows, tc.wantTotalRows)
}
}
}
// consumeRowIterator reads the schema and all values from a RowIterator and returns them.
func consumeRowIterator(it *RowIterator) ([][]Value, Schema, uint64, error) {
var (
got [][]Value
schema Schema
totalRows uint64
)
for {
var vls []Value
err := it.Next(&vls)
if err == iterator.Done {
return got, schema, totalRows, nil
}
if err != nil {
return got, schema, totalRows, err
}
got = append(got, vls)
schema = it.Schema
totalRows = it.TotalRows
}
}
func TestNextDuringErrorState(t *testing.T) {
pf := &pageFetcherStub{
fetchResponses: map[string]fetchResponse{
"": {err: errors.New("bang")},
},
}
it := newRowIterator(context.Background(), nil, pf.fetchPage)
var vals []Value
if err := it.Next(&vals); err == nil {
t.Errorf("Expected error after calling Next")
}
if err := it.Next(&vals); err == nil {
t.Errorf("Expected error calling Next again when iterator has a non-nil error.")
}
}
func TestNextAfterFinished(t *testing.T) {
testCases := []struct {
fetchResponses map[string]fetchResponse
want [][]Value
}{
{
fetchResponses: map[string]fetchResponse{
"": {
result: &fetchPageResult{
pageToken: "",
rows: [][]Value{{1, 2}, {11, 12}},
},
},
},
want: [][]Value{{1, 2}, {11, 12}},
},
{
fetchResponses: map[string]fetchResponse{
"": {
result: &fetchPageResult{
pageToken: "",
rows: [][]Value{},
},
},
},
want: [][]Value{},
},
}
for _, tc := range testCases {
pf := &pageFetcherStub{
fetchResponses: tc.fetchResponses,
}
it := newRowIterator(context.Background(), nil, pf.fetchPage)
values, _, _, err := consumeRowIterator(it)
if err != nil {
t.Fatal(err)
}
if (len(values) != 0 || len(tc.want) != 0) && !testutil.Equal(values, tc.want) {
t.Errorf("values: got:\n%v\nwant:\n%v", values, tc.want)
}
// Try calling Get again.
var vals []Value
if err := it.Next(&vals); err != iterator.Done {
t.Errorf("Expected Done calling Next when there are no more values")
}
}
}
func TestIteratorNextTypes(t *testing.T) {
it := newRowIterator(context.Background(), nil, nil)
for _, v := range []interface{}{3, "s", []int{}, &[]int{},
map[string]Value{}, &map[string]interface{}{},
struct{}{},
} {
if err := it.Next(v); err == nil {
t.Errorf("%v: want error, got nil", v)
}
}
}
func TestIteratorSourceJob(t *testing.T) {
testcases := []struct {
description string
src *rowSource
wantJob *Job
}{
{
description: "nil source",
src: nil,
wantJob: nil,
},
{
description: "empty source",
src: &rowSource{},
wantJob: nil,
},
{
description: "table source",
src: &rowSource{t: &Table{ProjectID: "p", DatasetID: "d", TableID: "t"}},
wantJob: nil,
},
{
description: "job source",
src: &rowSource{j: &Job{projectID: "p", location: "l", jobID: "j"}},
wantJob: &Job{projectID: "p", location: "l", jobID: "j"},
},
}
for _, tc := range testcases {
// Don't pass a page func, we're not reading from the iterator.
it := newRowIterator(context.Background(), tc.src, nil)
got := it.SourceJob()
// AllowUnexported because of the embedded client reference, which we're ignoring.
if !cmp.Equal(got, tc.wantJob, cmp.AllowUnexported(Job{})) {
t.Errorf("%s: mismatch on SourceJob, got %v want %v", tc.description, got, tc.wantJob)
}
}
}