| // Copyright 2014 Google LLC |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| |
| package datastore |
| |
| import ( |
| "context" |
| "encoding/json" |
| "errors" |
| "flag" |
| "fmt" |
| "log" |
| "os" |
| "reflect" |
| "sort" |
| "strings" |
| "sync" |
| "testing" |
| "time" |
| |
| "cloud.google.com/go/internal/testutil" |
| "cloud.google.com/go/internal/uid" |
| "cloud.google.com/go/rpcreplay" |
| "google.golang.org/api/iterator" |
| "google.golang.org/api/option" |
| "google.golang.org/grpc" |
| "google.golang.org/grpc/codes" |
| "google.golang.org/grpc/status" |
| ) |
| |
| // TODO(djd): Make test entity clean up more robust: some test entities may |
| // be left behind if tests are aborted, the transport fails, etc. |
| |
| var timeNow = time.Now() |
| |
| // suffix is a timestamp-based suffix which is appended to key names, |
| // particularly for the root keys of entity groups. This reduces flakiness |
| // when the tests are run in parallel. |
| var suffix string |
| |
| const replayFilename = "datastore.replay" |
| |
| type replayInfo struct { |
| ProjectID string |
| Time time.Time |
| } |
| |
| var ( |
| record = flag.Bool("record", false, "record RPCs") |
| |
| newTestClient = func(ctx context.Context, t *testing.T) *Client { |
| return newClient(ctx, t, nil) |
| } |
| ) |
| |
| func TestMain(m *testing.M) { |
| os.Exit(testMain(m)) |
| } |
| |
| func testMain(m *testing.M) int { |
| flag.Parse() |
| if testing.Short() { |
| if *record { |
| log.Fatal("cannot combine -short and -record") |
| } |
| if testutil.CanReplay(replayFilename) { |
| initReplay() |
| } |
| } else if *record { |
| if testutil.ProjID() == "" { |
| log.Fatal("must record with a project ID") |
| } |
| b, err := json.Marshal(replayInfo{ |
| ProjectID: testutil.ProjID(), |
| Time: timeNow, |
| }) |
| if err != nil { |
| log.Fatal(err) |
| } |
| rec, err := rpcreplay.NewRecorder(replayFilename, b) |
| if err != nil { |
| log.Fatal(err) |
| } |
| defer func() { |
| if err := rec.Close(); err != nil { |
| log.Fatalf("closing recorder: %v", err) |
| } |
| }() |
| newTestClient = func(ctx context.Context, t *testing.T) *Client { |
| return newClient(ctx, t, rec.DialOptions()) |
| } |
| log.Printf("recording to %s", replayFilename) |
| } |
| suffix = fmt.Sprintf("-t%d", timeNow.UnixNano()) |
| return m.Run() |
| } |
| |
| func initReplay() { |
| rep, err := rpcreplay.NewReplayer(replayFilename) |
| if err != nil { |
| log.Fatal(err) |
| } |
| defer rep.Close() |
| |
| var ri replayInfo |
| if err := json.Unmarshal(rep.Initial(), &ri); err != nil { |
| log.Fatalf("unmarshaling initial replay info: %v", err) |
| } |
| timeNow = ri.Time.In(time.Local) |
| |
| conn, err := rep.Connection() |
| if err != nil { |
| log.Fatal(err) |
| } |
| |
| newTestClient = func(ctx context.Context, t *testing.T) *Client { |
| grpcHeadersEnforcer := &testutil.HeadersEnforcer{ |
| OnFailure: t.Fatalf, |
| Checkers: []*testutil.HeaderChecker{ |
| testutil.XGoogClientHeaderChecker, |
| }, |
| } |
| |
| opts := append(grpcHeadersEnforcer.CallOptions(), option.WithGRPCConn(conn)) |
| client, err := NewClient(ctx, ri.ProjectID, opts...) |
| if err != nil { |
| t.Fatalf("NewClient: %v", err) |
| } |
| return client |
| } |
| log.Printf("replaying from %s", replayFilename) |
| } |
| |
| func newClient(ctx context.Context, t *testing.T, dialOpts []grpc.DialOption) *Client { |
| if testing.Short() { |
| t.Skip("Integration tests skipped in short mode") |
| } |
| ts := testutil.TokenSource(ctx, ScopeDatastore) |
| if ts == nil { |
| t.Skip("Integration tests skipped. See CONTRIBUTING.md for details") |
| } |
| |
| grpcHeadersEnforcer := &testutil.HeadersEnforcer{ |
| OnFailure: t.Fatalf, |
| Checkers: []*testutil.HeaderChecker{ |
| testutil.XGoogClientHeaderChecker, |
| }, |
| } |
| opts := append(grpcHeadersEnforcer.CallOptions(), option.WithTokenSource(ts)) |
| for _, opt := range dialOpts { |
| opts = append(opts, option.WithGRPCDialOption(opt)) |
| } |
| client, err := NewClient(ctx, testutil.ProjID(), opts...) |
| if err != nil { |
| t.Fatalf("NewClient: %v", err) |
| } |
| return client |
| } |
| |
| func TestIntegration_Basics(t *testing.T) { |
| ctx, _ := context.WithTimeout(context.Background(), time.Second*20) |
| client := newTestClient(ctx, t) |
| defer client.Close() |
| |
| type X struct { |
| I int |
| S string |
| T time.Time |
| U interface{} |
| } |
| |
| x0 := X{66, "99", timeNow.Truncate(time.Millisecond), "X"} |
| k, err := client.Put(ctx, IncompleteKey("BasicsX", nil), &x0) |
| if err != nil { |
| t.Fatalf("client.Put: %v", err) |
| } |
| x1 := X{} |
| err = client.Get(ctx, k, &x1) |
| if err != nil { |
| t.Errorf("client.Get: %v", err) |
| } |
| err = client.Delete(ctx, k) |
| if err != nil { |
| t.Errorf("client.Delete: %v", err) |
| } |
| if !testutil.Equal(x0, x1) { |
| t.Errorf("compare: x0=%v, x1=%v", x0, x1) |
| } |
| } |
| |
| func TestIntegration_TopLevelKeyLoaded(t *testing.T) { |
| ctx, _ := context.WithTimeout(context.Background(), time.Second*20) |
| client := newTestClient(ctx, t) |
| defer client.Close() |
| |
| completeKey := NameKey("EntityWithKey", "myent", nil) |
| |
| type EntityWithKey struct { |
| I int |
| S string |
| K *Key `datastore:"__key__"` |
| } |
| |
| in := &EntityWithKey{ |
| I: 12, |
| S: "abcd", |
| } |
| |
| k, err := client.Put(ctx, completeKey, in) |
| if err != nil { |
| t.Fatalf("client.Put: %v", err) |
| } |
| |
| var e EntityWithKey |
| err = client.Get(ctx, k, &e) |
| if err != nil { |
| t.Fatalf("client.Get: %v", err) |
| } |
| |
| // The two keys should be absolutely identical. |
| if !testutil.Equal(e.K, k) { |
| t.Fatalf("e.K not equal to k; got %#v, want %#v", e.K, k) |
| } |
| |
| } |
| |
| func TestIntegration_ListValues(t *testing.T) { |
| ctx := context.Background() |
| client := newTestClient(ctx, t) |
| defer client.Close() |
| |
| p0 := PropertyList{ |
| {Name: "L", Value: []interface{}{int64(12), "string", true}}, |
| } |
| k, err := client.Put(ctx, IncompleteKey("ListValue", nil), &p0) |
| if err != nil { |
| t.Fatalf("client.Put: %v", err) |
| } |
| var p1 PropertyList |
| if err := client.Get(ctx, k, &p1); err != nil { |
| t.Errorf("client.Get: %v", err) |
| } |
| if !testutil.Equal(p0, p1) { |
| t.Errorf("compare:\np0=%v\np1=%#v", p0, p1) |
| } |
| if err = client.Delete(ctx, k); err != nil { |
| t.Errorf("client.Delete: %v", err) |
| } |
| } |
| |
| func TestIntegration_GetMulti(t *testing.T) { |
| ctx := context.Background() |
| client := newTestClient(ctx, t) |
| defer client.Close() |
| |
| type X struct { |
| I int |
| } |
| p := NameKey("X", "x"+suffix, nil) |
| |
| cases := []struct { |
| key *Key |
| put bool |
| }{ |
| {key: NameKey("X", "item1", p), put: true}, |
| {key: NameKey("X", "item2", p), put: false}, |
| {key: NameKey("X", "item3", p), put: false}, |
| {key: NameKey("X", "item3", p), put: false}, |
| {key: NameKey("X", "item4", p), put: true}, |
| } |
| |
| var src, dst []*X |
| var srcKeys, dstKeys []*Key |
| for _, c := range cases { |
| dst = append(dst, &X{}) |
| dstKeys = append(dstKeys, c.key) |
| if c.put { |
| src = append(src, &X{}) |
| srcKeys = append(srcKeys, c.key) |
| } |
| } |
| if _, err := client.PutMulti(ctx, srcKeys, src); err != nil { |
| t.Error(err) |
| } |
| err := client.GetMulti(ctx, dstKeys, dst) |
| if err == nil { |
| t.Errorf("client.GetMulti got %v, expected error", err) |
| } |
| e, ok := err.(MultiError) |
| if !ok { |
| t.Errorf("client.GetMulti got %T, expected MultiError", err) |
| } |
| for i, err := range e { |
| got, want := err, (error)(nil) |
| if !cases[i].put { |
| got, want = err, ErrNoSuchEntity |
| } |
| if got != want { |
| t.Errorf("MultiError[%d] == %v, want %v", i, got, want) |
| } |
| } |
| } |
| |
| type Z struct { |
| S string |
| T string `datastore:",noindex"` |
| P []byte |
| K []byte `datastore:",noindex"` |
| } |
| |
| func (z Z) String() string { |
| var lens []string |
| v := reflect.ValueOf(z) |
| for i := 0; i < v.NumField(); i++ { |
| if l := v.Field(i).Len(); l > 0 { |
| lens = append(lens, fmt.Sprintf("len(%s)=%d", v.Type().Field(i).Name, l)) |
| } |
| } |
| return fmt.Sprintf("Z{ %s }", strings.Join(lens, ",")) |
| } |
| |
| func TestIntegration_UnindexableValues(t *testing.T) { |
| ctx := context.Background() |
| client := newTestClient(ctx, t) |
| defer client.Close() |
| |
| x1500 := strings.Repeat("x", 1500) |
| x1501 := strings.Repeat("x", 1501) |
| testCases := []struct { |
| in Z |
| wantErr bool |
| }{ |
| {in: Z{S: x1500}, wantErr: false}, |
| {in: Z{S: x1501}, wantErr: true}, |
| {in: Z{T: x1500}, wantErr: false}, |
| {in: Z{T: x1501}, wantErr: false}, |
| {in: Z{P: []byte(x1500)}, wantErr: false}, |
| {in: Z{P: []byte(x1501)}, wantErr: true}, |
| {in: Z{K: []byte(x1500)}, wantErr: false}, |
| {in: Z{K: []byte(x1501)}, wantErr: false}, |
| } |
| for _, tt := range testCases { |
| _, err := client.Put(ctx, IncompleteKey("BasicsZ", nil), &tt.in) |
| if (err != nil) != tt.wantErr { |
| t.Errorf("client.Put %s got err %v, want err %t", tt.in, err, tt.wantErr) |
| } |
| } |
| } |
| |
| func TestIntegration_NilKey(t *testing.T) { |
| ctx := context.Background() |
| client := newTestClient(ctx, t) |
| defer client.Close() |
| |
| testCases := []struct { |
| in K0 |
| wantErr bool |
| }{ |
| {in: K0{K: testKey0}, wantErr: false}, |
| {in: K0{}, wantErr: false}, |
| } |
| for _, tt := range testCases { |
| _, err := client.Put(ctx, IncompleteKey("NilKey", nil), &tt.in) |
| if (err != nil) != tt.wantErr { |
| t.Errorf("client.Put %s got err %v, want err %t", tt.in, err, tt.wantErr) |
| } |
| } |
| } |
| |
| type SQChild struct { |
| I, J int |
| T, U int64 |
| } |
| |
| type SQTestCase struct { |
| desc string |
| q *Query |
| wantCount int |
| wantSum int |
| } |
| |
| func testSmallQueries(ctx context.Context, t *testing.T, client *Client, parent *Key, children []*SQChild, |
| testCases []SQTestCase, extraTests ...func()) { |
| keys := make([]*Key, len(children)) |
| for i := range keys { |
| keys[i] = IncompleteKey("SQChild", parent) |
| } |
| keys, err := client.PutMulti(ctx, keys, children) |
| if err != nil { |
| t.Fatalf("client.PutMulti: %v", err) |
| } |
| defer func() { |
| err := client.DeleteMulti(ctx, keys) |
| if err != nil { |
| t.Errorf("client.DeleteMulti: %v", err) |
| } |
| }() |
| |
| for _, tc := range testCases { |
| count, err := client.Count(ctx, tc.q) |
| if err != nil { |
| t.Errorf("Count %q: %v", tc.desc, err) |
| continue |
| } |
| if count != tc.wantCount { |
| t.Errorf("Count %q: got %d want %d", tc.desc, count, tc.wantCount) |
| continue |
| } |
| } |
| |
| for _, tc := range testCases { |
| var got []SQChild |
| _, err := client.GetAll(ctx, tc.q, &got) |
| if err != nil { |
| t.Errorf("client.GetAll %q: %v", tc.desc, err) |
| continue |
| } |
| sum := 0 |
| for _, c := range got { |
| sum += c.I + c.J |
| } |
| if sum != tc.wantSum { |
| t.Errorf("sum %q: got %d want %d", tc.desc, sum, tc.wantSum) |
| continue |
| } |
| } |
| for _, x := range extraTests { |
| x() |
| } |
| } |
| |
| func TestIntegration_Filters(t *testing.T) { |
| ctx := context.Background() |
| client := newTestClient(ctx, t) |
| defer client.Close() |
| |
| parent := NameKey("SQParent", "TestIntegration_Filters"+suffix, nil) |
| now := timeNow.Truncate(time.Millisecond).Unix() |
| children := []*SQChild{ |
| {I: 0, T: now, U: now}, |
| {I: 1, T: now, U: now}, |
| {I: 2, T: now, U: now}, |
| {I: 3, T: now, U: now}, |
| {I: 4, T: now, U: now}, |
| {I: 5, T: now, U: now}, |
| {I: 6, T: now, U: now}, |
| {I: 7, T: now, U: now}, |
| } |
| baseQuery := NewQuery("SQChild").Ancestor(parent).Filter("T=", now) |
| testSmallQueries(ctx, t, client, parent, children, []SQTestCase{ |
| { |
| "I>1", |
| baseQuery.Filter("I>", 1), |
| 6, |
| 2 + 3 + 4 + 5 + 6 + 7, |
| }, |
| { |
| "I>2 AND I<=5", |
| baseQuery.Filter("I>", 2).Filter("I<=", 5), |
| 3, |
| 3 + 4 + 5, |
| }, |
| { |
| "I>=3 AND I<3", |
| baseQuery.Filter("I>=", 3).Filter("I<", 3), |
| 0, |
| 0, |
| }, |
| { |
| "I=4", |
| baseQuery.Filter("I=", 4), |
| 1, |
| 4, |
| }, |
| }, func() { |
| got := []*SQChild{} |
| want := []*SQChild{ |
| {I: 0, T: now, U: now}, |
| {I: 1, T: now, U: now}, |
| {I: 2, T: now, U: now}, |
| {I: 3, T: now, U: now}, |
| {I: 4, T: now, U: now}, |
| {I: 5, T: now, U: now}, |
| {I: 6, T: now, U: now}, |
| {I: 7, T: now, U: now}, |
| } |
| _, err := client.GetAll(ctx, baseQuery.Order("I"), &got) |
| if err != nil { |
| t.Errorf("client.GetAll: %v", err) |
| } |
| if !testutil.Equal(got, want) { |
| t.Errorf("compare: got=%v, want=%v", got, want) |
| } |
| }, func() { |
| got := []*SQChild{} |
| want := []*SQChild{ |
| {I: 7, T: now, U: now}, |
| {I: 6, T: now, U: now}, |
| {I: 5, T: now, U: now}, |
| {I: 4, T: now, U: now}, |
| {I: 3, T: now, U: now}, |
| {I: 2, T: now, U: now}, |
| {I: 1, T: now, U: now}, |
| {I: 0, T: now, U: now}, |
| } |
| _, err := client.GetAll(ctx, baseQuery.Order("-I"), &got) |
| if err != nil { |
| t.Errorf("client.GetAll: %v", err) |
| } |
| if !testutil.Equal(got, want) { |
| t.Errorf("compare: got=%v, want=%v", got, want) |
| } |
| }) |
| } |
| |
| type ckey struct{} |
| |
| func TestIntegration_LargeQuery(t *testing.T) { |
| ctx := context.Background() |
| client := newTestClient(ctx, t) |
| defer client.Close() |
| |
| parent := NameKey("LQParent", "TestIntegration_LargeQuery"+suffix, nil) |
| now := timeNow.Truncate(time.Millisecond).Unix() |
| |
| // Make a large number of children entities. |
| const n = 800 |
| children := make([]*SQChild, 0, n) |
| keys := make([]*Key, 0, n) |
| for i := 0; i < n; i++ { |
| children = append(children, &SQChild{I: i, T: now, U: now}) |
| keys = append(keys, IncompleteKey("SQChild", parent)) |
| } |
| |
| // Store using PutMulti in batches. |
| const batchSize = 500 |
| for i := 0; i < n; i = i + 500 { |
| j := i + batchSize |
| if j > n { |
| j = n |
| } |
| fullKeys, err := client.PutMulti(ctx, keys[i:j], children[i:j]) |
| if err != nil { |
| t.Fatalf("PutMulti(%d, %d): %v", i, j, err) |
| } |
| defer func() { |
| err := client.DeleteMulti(ctx, fullKeys) |
| if err != nil { |
| t.Errorf("client.DeleteMulti: %v", err) |
| } |
| }() |
| } |
| |
| q := NewQuery("SQChild").Ancestor(parent).Filter("T=", now).Order("I") |
| |
| // Wait group to allow us to run query tests in parallel below. |
| var wg sync.WaitGroup |
| |
| // Check we get the expected count and results for various limits/offsets. |
| queryTests := []struct { |
| limit, offset, want int |
| }{ |
| // Just limit. |
| {limit: 0, want: 0}, |
| {limit: 100, want: 100}, |
| {limit: 501, want: 501}, |
| {limit: n, want: n}, |
| {limit: n * 2, want: n}, |
| {limit: -1, want: n}, |
| // Just offset. |
| {limit: -1, offset: 100, want: n - 100}, |
| {limit: -1, offset: 500, want: n - 500}, |
| {limit: -1, offset: n, want: 0}, |
| // Limit and offset. |
| {limit: 100, offset: 100, want: 100}, |
| {limit: 1000, offset: 100, want: n - 100}, |
| {limit: 500, offset: 500, want: n - 500}, |
| } |
| for _, tt := range queryTests { |
| q := q.Limit(tt.limit).Offset(tt.offset) |
| wg.Add(1) |
| |
| go func(limit, offset, want int) { |
| defer wg.Done() |
| // Check Count returns the expected number of results. |
| count, err := client.Count(ctx, q) |
| if err != nil { |
| t.Errorf("client.Count(limit=%d offset=%d): %v", limit, offset, err) |
| return |
| } |
| if count != want { |
| t.Errorf("Count(limit=%d offset=%d) returned %d, want %d", limit, offset, count, want) |
| } |
| |
| var got []SQChild |
| _, err = client.GetAll(ctx, q, &got) |
| if err != nil { |
| t.Errorf("client.GetAll(limit=%d offset=%d): %v", limit, offset, err) |
| return |
| } |
| if len(got) != want { |
| t.Errorf("GetAll(limit=%d offset=%d) returned %d, want %d", limit, offset, len(got), want) |
| } |
| for i, child := range got { |
| if got, want := child.I, i+offset; got != want { |
| t.Errorf("GetAll(limit=%d offset=%d) got[%d].I == %d; want %d", limit, offset, i, got, want) |
| break |
| } |
| } |
| }(tt.limit, tt.offset, tt.want) |
| } |
| |
| // Also check iterator cursor behaviour. |
| cursorTests := []struct { |
| limit, offset int // Query limit and offset. |
| count int // The number of times to call "next" |
| want int // The I value of the desired element, -1 for "Done". |
| }{ |
| // No limits. |
| {count: 0, limit: -1, want: 0}, |
| {count: 5, limit: -1, want: 5}, |
| {count: 500, limit: -1, want: 500}, |
| {count: 1000, limit: -1, want: -1}, // No more results. |
| // Limits. |
| {count: 5, limit: 5, want: 5}, |
| {count: 500, limit: 5, want: 5}, |
| {count: 1000, limit: 1000, want: -1}, // No more results. |
| // Offsets. |
| {count: 0, offset: 5, limit: -1, want: 5}, |
| {count: 5, offset: 5, limit: -1, want: 10}, |
| {count: 200, offset: 500, limit: -1, want: 700}, |
| {count: 200, offset: 1000, limit: -1, want: -1}, // No more results. |
| } |
| for _, tt := range cursorTests { |
| wg.Add(1) |
| |
| go func(count, limit, offset, want int) { |
| defer wg.Done() |
| |
| ctx := context.WithValue(ctx, ckey{}, fmt.Sprintf("c=%d,l=%d,o=%d", count, limit, offset)) |
| // Run iterator through count calls to Next. |
| it := client.Run(ctx, q.Limit(limit).Offset(offset).KeysOnly()) |
| for i := 0; i < count; i++ { |
| _, err := it.Next(nil) |
| if err == iterator.Done { |
| break |
| } |
| if err != nil { |
| t.Errorf("count=%d, limit=%d, offset=%d: it.Next failed at i=%d", count, limit, offset, i) |
| return |
| } |
| } |
| |
| // Grab the cursor. |
| cursor, err := it.Cursor() |
| if err != nil { |
| t.Errorf("count=%d, limit=%d, offset=%d: it.Cursor: %v", count, limit, offset, err) |
| return |
| } |
| |
| // Make a request for the next element. |
| it = client.Run(ctx, q.Limit(1).Start(cursor)) |
| var entity SQChild |
| _, err = it.Next(&entity) |
| switch { |
| case want == -1: |
| if err != iterator.Done { |
| t.Errorf("count=%d, limit=%d, offset=%d: it.Next from cursor %v, want Done", count, limit, offset, err) |
| } |
| case err != nil: |
| t.Errorf("count=%d, limit=%d, offset=%d: it.Next from cursor: %v, want nil", count, limit, offset, err) |
| case entity.I != want: |
| t.Errorf("count=%d, limit=%d, offset=%d: got.I = %d, want %d", count, limit, offset, entity.I, want) |
| } |
| }(tt.count, tt.limit, tt.offset, tt.want) |
| } |
| wg.Wait() |
| } |
| |
| func TestIntegration_EventualConsistency(t *testing.T) { |
| // TODO(jba): either make this actually test eventual consistency, or |
| // delete it. Currently it behaves the same with or without the |
| // EventualConsistency call. |
| ctx := context.Background() |
| client := newTestClient(ctx, t) |
| defer client.Close() |
| |
| parent := NameKey("SQParent", "TestIntegration_EventualConsistency"+suffix, nil) |
| now := timeNow.Truncate(time.Millisecond).Unix() |
| children := []*SQChild{ |
| {I: 0, T: now, U: now}, |
| {I: 1, T: now, U: now}, |
| {I: 2, T: now, U: now}, |
| } |
| query := NewQuery("SQChild").Ancestor(parent).Filter("T =", now).EventualConsistency() |
| testSmallQueries(ctx, t, client, parent, children, nil, func() { |
| got, err := client.Count(ctx, query) |
| if err != nil { |
| t.Fatalf("Count: %v", err) |
| } |
| if got < 0 || 3 < got { |
| t.Errorf("Count: got %d, want [0,3]", got) |
| } |
| }) |
| } |
| |
| func TestIntegration_Projection(t *testing.T) { |
| ctx := context.Background() |
| client := newTestClient(ctx, t) |
| defer client.Close() |
| |
| parent := NameKey("SQParent", "TestIntegration_Projection"+suffix, nil) |
| now := timeNow.Truncate(time.Millisecond).Unix() |
| children := []*SQChild{ |
| {I: 1 << 0, J: 100, T: now, U: now}, |
| {I: 1 << 1, J: 100, T: now, U: now}, |
| {I: 1 << 2, J: 200, T: now, U: now}, |
| {I: 1 << 3, J: 300, T: now, U: now}, |
| {I: 1 << 4, J: 300, T: now, U: now}, |
| } |
| baseQuery := NewQuery("SQChild").Ancestor(parent).Filter("T=", now).Filter("J>", 150) |
| testSmallQueries(ctx, t, client, parent, children, []SQTestCase{ |
| { |
| "project", |
| baseQuery.Project("J"), |
| 3, |
| 200 + 300 + 300, |
| }, |
| { |
| "distinct", |
| baseQuery.Project("J").Distinct(), |
| 2, |
| 200 + 300, |
| }, |
| { |
| "distinct on", |
| baseQuery.Project("J").DistinctOn("J"), |
| 2, |
| 200 + 300, |
| }, |
| { |
| "project on meaningful (GD_WHEN) field", |
| baseQuery.Project("U"), |
| 3, |
| 0, |
| }, |
| }) |
| } |
| |
| func TestIntegration_AllocateIDs(t *testing.T) { |
| ctx := context.Background() |
| client := newTestClient(ctx, t) |
| defer client.Close() |
| |
| keys := make([]*Key, 5) |
| for i := range keys { |
| keys[i] = IncompleteKey("AllocID", nil) |
| } |
| keys, err := client.AllocateIDs(ctx, keys) |
| if err != nil { |
| t.Errorf("AllocID #0 failed: %v", err) |
| } |
| if want := len(keys); want != 5 { |
| t.Errorf("Expected to allocate 5 keys, %d keys are found", want) |
| } |
| for _, k := range keys { |
| if k.Incomplete() { |
| t.Errorf("Unexpeceted incomplete key found: %v", k) |
| } |
| } |
| } |
| |
| func TestIntegration_GetAllWithFieldMismatch(t *testing.T) { |
| ctx := context.Background() |
| client := newTestClient(ctx, t) |
| defer client.Close() |
| |
| type Fat struct { |
| X, Y int |
| } |
| type Thin struct { |
| X int |
| } |
| |
| // Ancestor queries (those within an entity group) are strongly consistent |
| // by default, which prevents a test from being flaky. |
| // See https://cloud.google.com/appengine/docs/go/datastore/queries#Go_Data_consistency |
| // for more information. |
| parent := NameKey("SQParent", "TestIntegration_GetAllWithFieldMismatch"+suffix, nil) |
| putKeys := make([]*Key, 3) |
| for i := range putKeys { |
| putKeys[i] = IDKey("GetAllThing", int64(10+i), parent) |
| _, err := client.Put(ctx, putKeys[i], &Fat{X: 20 + i, Y: 30 + i}) |
| if err != nil { |
| t.Fatalf("client.Put: %v", err) |
| } |
| } |
| |
| var got []Thin |
| want := []Thin{ |
| {X: 20}, |
| {X: 21}, |
| {X: 22}, |
| } |
| getKeys, err := client.GetAll(ctx, NewQuery("GetAllThing").Ancestor(parent), &got) |
| if len(getKeys) != 3 && !testutil.Equal(getKeys, putKeys) { |
| t.Errorf("client.GetAll: keys differ\ngetKeys=%v\nputKeys=%v", getKeys, putKeys) |
| } |
| if !testutil.Equal(got, want) { |
| t.Errorf("client.GetAll: entities differ\ngot =%v\nwant=%v", got, want) |
| } |
| if _, ok := err.(*ErrFieldMismatch); !ok { |
| t.Errorf("client.GetAll: got err=%v, want ErrFieldMismatch", err) |
| } |
| } |
| |
| func TestIntegration_KindlessQueries(t *testing.T) { |
| ctx := context.Background() |
| client := newTestClient(ctx, t) |
| defer client.Close() |
| |
| type Dee struct { |
| I int |
| Why string |
| } |
| type Dum struct { |
| I int |
| Pling string |
| } |
| |
| parent := NameKey("Tweedle", "tweedle"+suffix, nil) |
| |
| keys := []*Key{ |
| NameKey("Dee", "dee0", parent), |
| NameKey("Dum", "dum1", parent), |
| NameKey("Dum", "dum2", parent), |
| NameKey("Dum", "dum3", parent), |
| } |
| src := []interface{}{ |
| &Dee{1, "binary0001"}, |
| &Dum{2, "binary0010"}, |
| &Dum{4, "binary0100"}, |
| &Dum{8, "binary1000"}, |
| } |
| keys, err := client.PutMulti(ctx, keys, src) |
| if err != nil { |
| t.Fatalf("put: %v", err) |
| } |
| |
| testCases := []struct { |
| desc string |
| query *Query |
| want []int |
| wantErr string |
| }{ |
| { |
| desc: "Dee", |
| query: NewQuery("Dee"), |
| want: []int{1}, |
| }, |
| { |
| desc: "Doh", |
| query: NewQuery("Doh"), |
| want: nil}, |
| { |
| desc: "Dum", |
| query: NewQuery("Dum"), |
| want: []int{2, 4, 8}, |
| }, |
| { |
| desc: "", |
| query: NewQuery(""), |
| want: []int{1, 2, 4, 8}, |
| }, |
| { |
| desc: "Kindless filter", |
| query: NewQuery("").Filter("__key__ =", keys[2]), |
| want: []int{4}, |
| }, |
| { |
| desc: "Kindless order", |
| query: NewQuery("").Order("__key__"), |
| want: []int{1, 2, 4, 8}, |
| }, |
| { |
| desc: "Kindless bad filter", |
| query: NewQuery("").Filter("I =", 4), |
| wantErr: "kind is required", |
| }, |
| { |
| desc: "Kindless bad order", |
| query: NewQuery("").Order("-__key__"), |
| wantErr: "kind is required for all orders except __key__ ascending", |
| }, |
| } |
| for _, test := range testCases { |
| t.Run(test.desc, func(t *testing.T) { |
| q := test.query.Ancestor(parent) |
| gotCount, err := client.Count(ctx, q) |
| if err != nil { |
| if test.wantErr == "" || !strings.Contains(err.Error(), test.wantErr) { |
| t.Fatalf("count %q: err %v, want err %q", test.desc, err, test.wantErr) |
| } |
| return |
| } |
| if test.wantErr != "" { |
| t.Fatalf("count %q: want err %q", test.desc, test.wantErr) |
| } |
| if gotCount != len(test.want) { |
| t.Fatalf("count %q: got %d want %d", test.desc, gotCount, len(test.want)) |
| } |
| var got []int |
| for iter := client.Run(ctx, q); ; { |
| var dst struct { |
| I int |
| Why, Pling string |
| } |
| _, err := iter.Next(&dst) |
| if err == iterator.Done { |
| break |
| } |
| if err != nil { |
| t.Fatalf("iter.Next %q: %v", test.desc, err) |
| } |
| got = append(got, dst.I) |
| } |
| sort.Ints(got) |
| if !testutil.Equal(got, test.want) { |
| t.Fatalf("elems %q: got %+v want %+v", test.desc, got, test.want) |
| } |
| }) |
| } |
| } |
| |
| func TestIntegration_Transaction(t *testing.T) { |
| ctx := context.Background() |
| client := newTestClient(ctx, t) |
| defer client.Close() |
| |
| type Counter struct { |
| N int |
| T time.Time |
| } |
| |
| bangErr := errors.New("bang") |
| tests := []struct { |
| desc string |
| causeConflict []bool |
| retErr []error |
| want int |
| wantErr error |
| }{ |
| { |
| desc: "3 attempts, no conflicts", |
| causeConflict: []bool{false}, |
| retErr: []error{nil}, |
| want: 11, |
| }, |
| { |
| desc: "1 attempt, user error", |
| causeConflict: []bool{false}, |
| retErr: []error{bangErr}, |
| wantErr: bangErr, |
| }, |
| { |
| desc: "2 attempts, 1 conflict", |
| causeConflict: []bool{true, false}, |
| retErr: []error{nil, nil}, |
| want: 13, // Each conflict increments by 2. |
| }, |
| { |
| desc: "3 attempts, 3 conflicts", |
| causeConflict: []bool{true, true, true}, |
| retErr: []error{nil, nil, nil}, |
| wantErr: ErrConcurrentTransaction, |
| }, |
| } |
| for i, test := range tests { |
| t.Run(test.desc, func(t *testing.T) { |
| // Put a new counter. |
| c := &Counter{N: 10, T: timeNow} |
| key, err := client.Put(ctx, IncompleteKey("TransCounter", nil), c) |
| if err != nil { |
| t.Fatal(err) |
| } |
| defer client.Delete(ctx, key) |
| |
| // Increment the counter in a transaction. |
| // The test case can manually cause a conflict or return an |
| // error at each attempt. |
| var attempts int |
| _, err = client.RunInTransaction(ctx, func(tx *Transaction) error { |
| attempts++ |
| if attempts > len(test.causeConflict) { |
| return fmt.Errorf("too many attempts. Got %d, max %d", attempts, len(test.causeConflict)) |
| } |
| |
| var c Counter |
| if err := tx.Get(key, &c); err != nil { |
| return err |
| } |
| c.N++ |
| if _, err := tx.Put(key, &c); err != nil { |
| return err |
| } |
| |
| if test.causeConflict[attempts-1] { |
| c.N++ |
| if _, err := client.Put(ctx, key, &c); err != nil { |
| return err |
| } |
| } |
| |
| return test.retErr[attempts-1] |
| }, MaxAttempts(i)) |
| |
| // Check the error returned by RunInTransaction. |
| if err != test.wantErr { |
| t.Fatalf("got err %v, want %v", err, test.wantErr) |
| } |
| if test.wantErr != nil { |
| // If we were expecting an error, this is where the test ends. |
| return |
| } |
| |
| // Check the final value of the counter. |
| if err := client.Get(ctx, key, c); err != nil { |
| t.Fatal(err) |
| } |
| if c.N != test.want { |
| t.Fatalf("counter N=%d, want N=%d", c.N, test.want) |
| } |
| }) |
| } |
| } |
| |
| func TestIntegration_ReadOnlyTransaction(t *testing.T) { |
| if testing.Short() { |
| t.Skip("Integration tests skipped in short mode") |
| } |
| ctx := context.Background() |
| client := newClient(ctx, t, nil) |
| defer client.Close() |
| |
| type value struct{ N int } |
| |
| // Put a value. |
| const n = 5 |
| v := &value{N: n} |
| key, err := client.Put(ctx, IncompleteKey("roTxn", nil), v) |
| if err != nil { |
| t.Fatal(err) |
| } |
| defer client.Delete(ctx, key) |
| |
| // Read it from a read-only transaction. |
| _, err = client.RunInTransaction(ctx, func(tx *Transaction) error { |
| if err := tx.Get(key, v); err != nil { |
| return err |
| } |
| return nil |
| }, ReadOnly) |
| if err != nil { |
| t.Fatal(err) |
| } |
| if v.N != n { |
| t.Fatalf("got %d, want %d", v.N, n) |
| } |
| |
| // Attempting to write from a read-only transaction is an error. |
| _, err = client.RunInTransaction(ctx, func(tx *Transaction) error { |
| if _, err := tx.Put(key, v); err != nil { |
| return err |
| } |
| return nil |
| }, ReadOnly) |
| if err == nil { |
| t.Fatal("got nil, want error") |
| } |
| } |
| |
| func TestIntegration_NilPointers(t *testing.T) { |
| ctx := context.Background() |
| client := newTestClient(ctx, t) |
| defer client.Close() |
| |
| type X struct { |
| S string |
| } |
| |
| src := []*X{{"zero"}, {"one"}} |
| keys := []*Key{IncompleteKey("NilX", nil), IncompleteKey("NilX", nil)} |
| keys, err := client.PutMulti(ctx, keys, src) |
| if err != nil { |
| t.Fatalf("PutMulti: %v", err) |
| } |
| |
| // It's okay to store into a slice of nil *X. |
| xs := make([]*X, 2) |
| if err := client.GetMulti(ctx, keys, xs); err != nil { |
| t.Errorf("GetMulti: %v", err) |
| } else if !testutil.Equal(xs, src) { |
| t.Errorf("GetMulti fetched %v, want %v", xs, src) |
| } |
| |
| // It isn't okay to store into a single nil *X. |
| var x0 *X |
| if err, want := client.Get(ctx, keys[0], x0), ErrInvalidEntityType; err != want { |
| t.Errorf("Get: err %v; want %v", err, want) |
| } |
| |
| // Test that deleting with duplicate keys work. |
| keys = append(keys, keys...) |
| if err := client.DeleteMulti(ctx, keys); err != nil { |
| t.Errorf("Delete: %v", err) |
| } |
| } |
| |
| func TestIntegration_NestedRepeatedElementNoIndex(t *testing.T) { |
| ctx := context.Background() |
| client := newTestClient(ctx, t) |
| defer client.Close() |
| |
| type Inner struct { |
| Name string |
| Value string `datastore:",noindex"` |
| } |
| type Outer struct { |
| Config []Inner |
| } |
| m := &Outer{ |
| Config: []Inner{ |
| {Name: "short", Value: "a"}, |
| {Name: "long", Value: strings.Repeat("a", 2000)}, |
| }, |
| } |
| |
| key := NameKey("Nested", "Nested"+suffix, nil) |
| if _, err := client.Put(ctx, key, m); err != nil { |
| t.Fatalf("client.Put: %v", err) |
| } |
| if err := client.Delete(ctx, key); err != nil { |
| t.Fatalf("client.Delete: %v", err) |
| } |
| } |
| |
| func TestIntegration_PointerFields(t *testing.T) { |
| ctx := context.Background() |
| client := newTestClient(ctx, t) |
| defer client.Close() |
| |
| want := populatedPointers() |
| key, err := client.Put(ctx, IncompleteKey("pointers", nil), want) |
| if err != nil { |
| t.Fatal(err) |
| } |
| var got Pointers |
| if err := client.Get(ctx, key, &got); err != nil { |
| t.Fatal(err) |
| } |
| if got.Pi == nil || *got.Pi != *want.Pi { |
| t.Errorf("Pi: got %v, want %v", got.Pi, *want.Pi) |
| } |
| if got.Ps == nil || *got.Ps != *want.Ps { |
| t.Errorf("Ps: got %v, want %v", got.Ps, *want.Ps) |
| } |
| if got.Pb == nil || *got.Pb != *want.Pb { |
| t.Errorf("Pb: got %v, want %v", got.Pb, *want.Pb) |
| } |
| if got.Pf == nil || *got.Pf != *want.Pf { |
| t.Errorf("Pf: got %v, want %v", got.Pf, *want.Pf) |
| } |
| if got.Pg == nil || *got.Pg != *want.Pg { |
| t.Errorf("Pg: got %v, want %v", got.Pg, *want.Pg) |
| } |
| if got.Pt == nil || !got.Pt.Equal(*want.Pt) { |
| t.Errorf("Pt: got %v, want %v", got.Pt, *want.Pt) |
| } |
| } |
| |
| func TestIntegration_Mutate(t *testing.T) { |
| // test Client.Mutate |
| testMutate(t, func(ctx context.Context, client *Client, muts ...*Mutation) ([]*Key, error) { |
| return client.Mutate(ctx, muts...) |
| }) |
| // test Transaction.Mutate |
| testMutate(t, func(ctx context.Context, client *Client, muts ...*Mutation) ([]*Key, error) { |
| var pkeys []*PendingKey |
| commit, err := client.RunInTransaction(ctx, func(tx *Transaction) error { |
| var err error |
| pkeys, err = tx.Mutate(muts...) |
| return err |
| }) |
| if err != nil { |
| return nil, err |
| } |
| var keys []*Key |
| for _, pk := range pkeys { |
| keys = append(keys, commit.Key(pk)) |
| } |
| return keys, nil |
| }) |
| } |
| |
| func testMutate(t *testing.T, mutate func(ctx context.Context, client *Client, muts ...*Mutation) ([]*Key, error)) { |
| ctx := context.Background() |
| client := newTestClient(ctx, t) |
| defer client.Close() |
| |
| type T struct{ I int } |
| |
| check := func(k *Key, want interface{}) { |
| var x T |
| err := client.Get(ctx, k, &x) |
| switch want := want.(type) { |
| case error: |
| if err != want { |
| t.Errorf("key %s: got error %v, want %v", k, err, want) |
| } |
| case int: |
| if err != nil { |
| t.Fatalf("key %s: %v", k, err) |
| } |
| if x.I != want { |
| t.Errorf("key %s: got %d, want %d", k, x.I, want) |
| } |
| default: |
| panic("check: bad arg") |
| } |
| } |
| |
| keys, err := mutate(ctx, client, |
| NewInsert(IncompleteKey("t", nil), &T{1}), |
| NewUpsert(IncompleteKey("t", nil), &T{2}), |
| ) |
| if err != nil { |
| t.Fatal(err) |
| } |
| check(keys[0], 1) |
| check(keys[1], 2) |
| |
| _, err = mutate(ctx, client, |
| NewUpdate(keys[0], &T{3}), |
| NewDelete(keys[1]), |
| ) |
| if err != nil { |
| t.Fatal(err) |
| } |
| check(keys[0], 3) |
| check(keys[1], ErrNoSuchEntity) |
| |
| _, err = mutate(ctx, client, NewInsert(keys[0], &T{4})) |
| if got, want := status.Code(err), codes.AlreadyExists; got != want { |
| t.Errorf("Insert existing key: got %s, want %s", got, want) |
| } |
| |
| _, err = mutate(ctx, client, NewUpdate(keys[1], &T{4})) |
| if got, want := status.Code(err), codes.NotFound; got != want { |
| t.Errorf("Update non-existing key: got %s, want %s", got, want) |
| } |
| } |
| |
| func TestIntegration_DetectProjectID(t *testing.T) { |
| if testing.Short() { |
| t.Skip("Integration tests skipped in short mode") |
| } |
| ctx := context.Background() |
| |
| creds := testutil.Credentials(ctx, ScopeDatastore) |
| if creds == nil { |
| t.Skip("Integration tests skipped. See CONTRIBUTING.md for details") |
| } |
| |
| // Use creds with project ID. |
| if _, err := NewClient(ctx, DetectProjectID, option.WithCredentials(creds)); err != nil { |
| t.Errorf("NewClient: %v", err) |
| } |
| |
| ts := testutil.ErroringTokenSource{} |
| // Try to use creds without project ID. |
| _, err := NewClient(ctx, DetectProjectID, option.WithTokenSource(ts)) |
| if err == nil || err.Error() != "datastore: see the docs on DetectProjectID" { |
| t.Errorf("expected an error while using TokenSource that does not have a project ID") |
| } |
| } |
| |
| var genKeyName = uid.NewSpace("datastore-integration", nil) |
| |
| func TestIntegration_Project_TimestampStoreAndRetrieve(t *testing.T) { |
| ctx := context.Background() |
| client := newTestClient(ctx, t) |
| defer client.Close() |
| |
| type T struct{ Created time.Time } |
| |
| keyName := genKeyName.New() |
| |
| now := time.Now() |
| k, err := client.Put(ctx, IncompleteKey(keyName, nil), &T{Created: now}) |
| if err != nil { |
| t.Fatal(err) |
| } |
| defer func() { |
| if err := client.Delete(ctx, k); err != nil { |
| log.Println(err) |
| } |
| }() |
| |
| // Without .Ancestor, this query is eventually consistent (so this test |
| // would be flakey). Ancestor queries, however, are strongly consistent. |
| // See more at https://cloud.google.com/datastore/docs/articles/balancing-strong-and-eventual-consistency-with-google-cloud-datastore/. |
| q := NewQuery(k.Kind).Ancestor(k) |
| res := []T{} |
| if _, err := client.GetAll(ctx, q, &res); err != nil { |
| t.Fatal(err) |
| } |
| if len(res) != 1 { |
| t.Fatalf("expected 1 result, got %d", len(res)) |
| } |
| if got, want := res[0].Created.Unix(), now.Unix(); got != want { |
| t.Fatalf("got %v, want %v", got, want) |
| } |
| } |