bigquery/table_test.go

// Copyright 2017 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package bigquery

import (
	"testing"
	"time"

	"cloud.google.com/go/internal/testutil"
	bq "google.golang.org/api/bigquery/v2"
)

func TestBQToTableMetadata(t *testing.T) {
	aTime := time.Date(2017, 1, 26, 0, 0, 0, 0, time.Local)
	aTimeMillis := aTime.UnixNano() / 1e6
	for _, test := range []struct {
		in   *bq.Table
		want *TableMetadata
	}{
		{&bq.Table{}, &TableMetadata{}}, // test minimal case
		{
			&bq.Table{
				CreationTime:     aTimeMillis,
				Description:      "desc",
				Etag:             "etag",
				ExpirationTime:   aTimeMillis,
				FriendlyName:     "fname",
				Id:               "id",
				LastModifiedTime: uint64(aTimeMillis),
				Location:         "loc",
				NumBytes:         123,
				NumLongTermBytes: 23,
				NumRows:          7,
				StreamingBuffer: &bq.Streamingbuffer{
					EstimatedBytes:  11,
					EstimatedRows:   3,
					OldestEntryTime: uint64(aTimeMillis),
				},
				TimePartitioning: &bq.TimePartitioning{
					ExpirationMs: 7890,
					Type:         "DAY",
					Field:        "pfield",
				},
				Clustering: &bq.Clustering{
					Fields: []string{"cfield1", "cfield2"},
				},
				RequirePartitionFilter:  true,
				EncryptionConfiguration: &bq.EncryptionConfiguration{KmsKeyName: "keyName"},
				Type:                    "EXTERNAL",
				View:                    &bq.ViewDefinition{Query: "view-query"},
				Labels:                  map[string]string{"a": "b"},
				ExternalDataConfiguration: &bq.ExternalDataConfiguration{
					SourceFormat: "GOOGLE_SHEETS",
				},
			},
			&TableMetadata{
				Description:        "desc",
				Name:               "fname",
				Location:           "loc",
				ViewQuery:          "view-query",
				FullID:             "id",
				Type:               ExternalTable,
				Labels:             map[string]string{"a": "b"},
				ExternalDataConfig: &ExternalDataConfig{SourceFormat: GoogleSheets},
				ExpirationTime:     aTime.Truncate(time.Millisecond),
				CreationTime:       aTime.Truncate(time.Millisecond),
				LastModifiedTime:   aTime.Truncate(time.Millisecond),
				NumBytes:           123,
				NumLongTermBytes:   23,
				NumRows:            7,
				TimePartitioning: &TimePartitioning{
					Expiration: 7890 * time.Millisecond,
					Field:      "pfield",
				},
				Clustering: &Clustering{
					Fields: []string{"cfield1", "cfield2"},
				},
				RequirePartitionFilter: true,
				StreamingBuffer: &StreamingBuffer{
					EstimatedBytes:  11,
					EstimatedRows:   3,
					OldestEntryTime: aTime,
				},
				EncryptionConfig: &EncryptionConfig{KMSKeyName: "keyName"},
				ETag:             "etag",
			},
		},
	} {
		got, err := bqToTableMetadata(test.in)
		if err != nil {
			t.Fatal(err)
		}
		if diff := testutil.Diff(got, test.want); diff != "" {
			t.Errorf("%+v:\n, -got, +want:\n%s", test.in, diff)
		}
	}
}

func TestTableMetadataToBQ(t *testing.T) {
	aTime := time.Date(2017, 1, 26, 0, 0, 0, 0, time.Local)
	aTimeMillis := aTime.UnixNano() / 1e6
	sc := Schema{fieldSchema("desc", "name", "STRING", false, true)}

	for _, test := range []struct {
		in   *TableMetadata
		want *bq.Table
	}{
		{nil, &bq.Table{}},
		{&TableMetadata{}, &bq.Table{}},
		{
			&TableMetadata{
				Name:               "n",
				Description:        "d",
				Schema:             sc,
				ExpirationTime:     aTime,
				Labels:             map[string]string{"a": "b"},
				ExternalDataConfig: &ExternalDataConfig{SourceFormat: Bigtable},
				EncryptionConfig:   &EncryptionConfig{KMSKeyName: "keyName"},
			},
			&bq.Table{
				FriendlyName: "n",
				Description:  "d",
				Schema: &bq.TableSchema{
					Fields: []*bq.TableFieldSchema{
						bqTableFieldSchema("desc", "name", "STRING", "REQUIRED"),
					},
				},
				ExpirationTime:            aTimeMillis,
				Labels:                    map[string]string{"a": "b"},
				ExternalDataConfiguration: &bq.ExternalDataConfiguration{SourceFormat: "BIGTABLE"},
				EncryptionConfiguration:   &bq.EncryptionConfiguration{KmsKeyName: "keyName"},
			},
		},
		{
			&TableMetadata{ViewQuery: "q"},
			&bq.Table{
				View: &bq.ViewDefinition{
					Query:           "q",
					UseLegacySql:    false,
					ForceSendFields: []string{"UseLegacySql"},
				},
			},
		},
		{
			&TableMetadata{
				ViewQuery:              "q",
				UseLegacySQL:           true,
				TimePartitioning:       &TimePartitioning{},
				RequirePartitionFilter: true,
			},
			&bq.Table{
				View: &bq.ViewDefinition{
					Query:        "q",
					UseLegacySql: true,
				},
				TimePartitioning: &bq.TimePartitioning{
					Type:         "DAY",
					ExpirationMs: 0,
				},
				RequirePartitionFilter: true,
			},
		},
		{
			&TableMetadata{
				ViewQuery:      "q",
				UseStandardSQL: true,
				TimePartitioning: &TimePartitioning{
					Expiration: time.Second,
					Field:      "ofDreams",
				},
				Clustering: &Clustering{
					Fields: []string{"cfield1"},
				},
			},
			&bq.Table{
				View: &bq.ViewDefinition{
					Query:           "q",
					UseLegacySql:    false,
					ForceSendFields: []string{"UseLegacySql"},
				},
				TimePartitioning: &bq.TimePartitioning{
					Type:         "DAY",
					ExpirationMs: 1000,
					Field:        "ofDreams",
				},
				Clustering: &bq.Clustering{
					Fields: []string{"cfield1"},
				},
			},
		},
		{
			&TableMetadata{
				RangePartitioning: &RangePartitioning{
					Field: "ofNumbers",
					Range: &RangePartitioningRange{
						Start:    1,
						End:      100,
						Interval: 5,
					},
				},
				Clustering: &Clustering{
					Fields: []string{"cfield1"},
				},
			},
			&bq.Table{

				RangePartitioning: &bq.RangePartitioning{
					Field: "ofNumbers",
					Range: &bq.RangePartitioningRange{
						Start:    1,
						End:      100,
						Interval: 5,
					},
				},
				Clustering: &bq.Clustering{
					Fields: []string{"cfield1"},
				},
			},
		},
		{
			&TableMetadata{ExpirationTime: NeverExpire},
			&bq.Table{ExpirationTime: 0},
		},
	} {
		got, err := test.in.toBQ()
		if err != nil {
			t.Fatalf("%+v: %v", test.in, err)
		}
		if diff := testutil.Diff(got, test.want); diff != "" {
			t.Errorf("%+v:\n-got, +want:\n%s", test.in, diff)
		}
	}

	// Errors
	for _, in := range []*TableMetadata{
		{Schema: sc, ViewQuery: "q"}, // can't have both schema and query
		{UseLegacySQL: true},         // UseLegacySQL without query
		{UseStandardSQL: true},       // UseStandardSQL without query
		// read-only fields
		{FullID: "x"},
		{Type: "x"},
		{CreationTime: aTime},
		{LastModifiedTime: aTime},
		{NumBytes: 1},
		{NumLongTermBytes: 1},
		{NumRows: 1},
		{StreamingBuffer: &StreamingBuffer{}},
		{ETag: "x"},
		// expiration time outside allowable range is invalid
		// See https://godoc.org/time#Time.UnixNano
		{ExpirationTime: time.Date(1677, 9, 21, 0, 12, 43, 145224192, time.UTC).Add(-1)},
		{ExpirationTime: time.Date(2262, 04, 11, 23, 47, 16, 854775807, time.UTC).Add(1)},
	} {
		_, err := in.toBQ()
		if err == nil {
			t.Errorf("%+v: got nil, want error", in)
		}
	}
}

func TestTableMetadataToUpdateToBQ(t *testing.T) {
	aTime := time.Date(2017, 1, 26, 0, 0, 0, 0, time.Local)
	for _, test := range []struct {
		tm   TableMetadataToUpdate
		want *bq.Table
	}{
		{
			tm:   TableMetadataToUpdate{},
			want: &bq.Table{},
		},
		{
			tm: TableMetadataToUpdate{
				Description: "d",
				Name:        "n",
			},
			want: &bq.Table{
				Description:     "d",
				FriendlyName:    "n",
				ForceSendFields: []string{"Description", "FriendlyName"},
			},
		},
		{
			tm: TableMetadataToUpdate{
				Schema:         Schema{fieldSchema("desc", "name", "STRING", false, true)},
				ExpirationTime: aTime,
			},
			want: &bq.Table{
				Schema: &bq.TableSchema{
					Fields: []*bq.TableFieldSchema{
						bqTableFieldSchema("desc", "name", "STRING", "REQUIRED"),
					},
				},
				ExpirationTime:  aTime.UnixNano() / 1e6,
				ForceSendFields: []string{"Schema", "ExpirationTime"},
			},
		},
		{
			tm: TableMetadataToUpdate{ViewQuery: "q"},
			want: &bq.Table{
				View: &bq.ViewDefinition{Query: "q", ForceSendFields: []string{"Query"}},
			},
		},
		{
			tm: TableMetadataToUpdate{UseLegacySQL: false},
			want: &bq.Table{
				View: &bq.ViewDefinition{
					UseLegacySql:    false,
					ForceSendFields: []string{"UseLegacySql"},
				},
			},
		},
		{
			tm: TableMetadataToUpdate{ViewQuery: "q", UseLegacySQL: true},
			want: &bq.Table{
				View: &bq.ViewDefinition{
					Query:           "q",
					UseLegacySql:    true,
					ForceSendFields: []string{"Query", "UseLegacySql"},
				},
			},
		},
		{
			tm: func() (tm TableMetadataToUpdate) {
				tm.SetLabel("L", "V")
				tm.DeleteLabel("D")
				return tm
			}(),
			want: &bq.Table{
				Labels:     map[string]string{"L": "V"},
				NullFields: []string{"Labels.D"},
			},
		},
		{
			tm: TableMetadataToUpdate{ExpirationTime: NeverExpire},
			want: &bq.Table{
				NullFields: []string{"ExpirationTime"},
			},
		},
		{
			tm: TableMetadataToUpdate{TimePartitioning: &TimePartitioning{Expiration: 0}},
			want: &bq.Table{
				TimePartitioning: &bq.TimePartitioning{
					Type:            "DAY",
					ForceSendFields: []string{"RequirePartitionFilter"},
					NullFields:      []string{"ExpirationMs"},
				},
			},
		},
		{
			tm: TableMetadataToUpdate{TimePartitioning: &TimePartitioning{Expiration: time.Duration(time.Hour)}},
			want: &bq.Table{
				TimePartitioning: &bq.TimePartitioning{
					ExpirationMs:    3600000,
					Type:            "DAY",
					ForceSendFields: []string{"RequirePartitionFilter"},
				},
			},
		},
		{
			tm: TableMetadataToUpdate{RequirePartitionFilter: false},
			want: &bq.Table{
				RequirePartitionFilter: false,
				ForceSendFields:        []string{"RequirePartitionFilter"},
			},
		},
		{
			tm: TableMetadataToUpdate{RequirePartitionFilter: true},
			want: &bq.Table{
				RequirePartitionFilter: true,
				ForceSendFields:        []string{"RequirePartitionFilter"},
			},
		},
	} {
		got, _ := test.tm.toBQ()
		if !testutil.Equal(got, test.want) {
			t.Errorf("%+v:\ngot  %+v\nwant %+v", test.tm, got, test.want)
		}
	}
}

func TestTableMetadataToUpdateToBQErrors(t *testing.T) {
	// See https://godoc.org/time#Time.UnixNano
	start := time.Date(1677, 9, 21, 0, 12, 43, 145224192, time.UTC)
	end := time.Date(2262, 04, 11, 23, 47, 16, 854775807, time.UTC)

	for _, test := range []struct {
		desc    string
		aTime   time.Time
		wantErr bool
	}{
		{desc: "ignored zero value", aTime: time.Time{}, wantErr: false},
		{desc: "earliest valid time", aTime: start, wantErr: false},
		{desc: "latested valid time", aTime: end, wantErr: false},
		{desc: "invalid times before 1678", aTime: start.Add(-1), wantErr: true},
		{desc: "invalid times after 2262", aTime: end.Add(1), wantErr: true},
		{desc: "valid times after 1678", aTime: start.Add(1), wantErr: false},
		{desc: "valid times before 2262", aTime: end.Add(-1), wantErr: false},
	} {
		tm := &TableMetadataToUpdate{ExpirationTime: test.aTime}
		_, err := tm.toBQ()
		if test.wantErr && err == nil {
			t.Errorf("[%s] got no error, want error", test.desc)
		}
		if !test.wantErr && err != nil {
			t.Errorf("[%s] got error, want no error", test.desc)
		}
	}
}