| // Copyright 2018 Google LLC |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| |
| package storage |
| |
| import ( |
| "errors" |
| "fmt" |
| "io" |
| "io/ioutil" |
| "net/http" |
| "strconv" |
| "strings" |
| "testing" |
| |
| "golang.org/x/net/context" |
| "google.golang.org/api/option" |
| ) |
| |
| const readData = "0123456789" |
| |
| func TestRangeReader(t *testing.T) { |
| hc, close := newTestServer(handleRangeRead) |
| defer close() |
| ctx := context.Background() |
| c, err := NewClient(ctx, option.WithHTTPClient(hc)) |
| if err != nil { |
| t.Fatal(err) |
| } |
| obj := c.Bucket("b").Object("o") |
| for _, test := range []struct { |
| offset, length int64 |
| want string |
| }{ |
| {0, -1, readData}, |
| {0, 10, readData}, |
| {0, 5, readData[:5]}, |
| {1, 3, readData[1:4]}, |
| {6, -1, readData[6:]}, |
| {4, 20, readData[4:]}, |
| } { |
| r, err := obj.NewRangeReader(ctx, test.offset, test.length) |
| if err != nil { |
| t.Errorf("%d/%d: %v", test.offset, test.length, err) |
| continue |
| } |
| gotb, err := ioutil.ReadAll(r) |
| if err != nil { |
| t.Errorf("%d/%d: %v", test.offset, test.length, err) |
| continue |
| } |
| if got := string(gotb); got != test.want { |
| t.Errorf("%d/%d: got %q, want %q", test.offset, test.length, got, test.want) |
| } |
| } |
| } |
| |
| func handleRangeRead(w http.ResponseWriter, r *http.Request) { |
| rh := strings.TrimSpace(r.Header.Get("Range")) |
| data := readData |
| var from, to int |
| if rh == "" { |
| from = 0 |
| to = len(data) |
| } else { |
| // assume "bytes=N-" or "bytes=N-M" |
| var err error |
| i := strings.IndexRune(rh, '=') |
| j := strings.IndexRune(rh, '-') |
| from, err = strconv.Atoi(rh[i+1 : j]) |
| if err != nil { |
| w.WriteHeader(500) |
| return |
| } |
| to = len(data) |
| if j+1 < len(rh) { |
| to, err = strconv.Atoi(rh[j+1:]) |
| if err != nil { |
| w.WriteHeader(500) |
| return |
| } |
| to++ // Range header is inclusive, Go slice is exclusive |
| } |
| if from >= len(data) && to != from { |
| w.WriteHeader(416) |
| return |
| } |
| if from > len(data) { |
| from = len(data) |
| } |
| if to > len(data) { |
| to = len(data) |
| } |
| } |
| data = data[from:to] |
| if data != readData { |
| w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", from, to-1, len(readData))) |
| w.WriteHeader(http.StatusPartialContent) |
| } |
| if _, err := w.Write([]byte(data)); err != nil { |
| panic(err) |
| } |
| } |
| |
| type http2Error string |
| |
| func (h http2Error) Error() string { |
| return string(h) |
| } |
| |
| func TestRangeReaderRetry(t *testing.T) { |
| retryErr := http2Error("blah blah INTERNAL_ERROR") |
| readBytes := []byte(readData) |
| hc, close := newTestServer(handleRangeRead) |
| defer close() |
| ctx := context.Background() |
| c, err := NewClient(ctx, option.WithHTTPClient(hc)) |
| if err != nil { |
| t.Fatal(err) |
| } |
| |
| obj := c.Bucket("b").Object("o") |
| for i, test := range []struct { |
| offset, length int64 |
| bodies []fakeReadCloser |
| want string |
| }{ |
| { |
| offset: 0, |
| length: -1, |
| bodies: []fakeReadCloser{ |
| {data: readBytes, counts: []int{10}, err: io.EOF}, |
| }, |
| want: readData, |
| }, |
| { |
| offset: 0, |
| length: -1, |
| bodies: []fakeReadCloser{ |
| {data: readBytes, counts: []int{3}, err: retryErr}, |
| {data: readBytes[3:], counts: []int{5, 2}, err: io.EOF}, |
| }, |
| want: readData, |
| }, |
| { |
| offset: 0, |
| length: -1, |
| bodies: []fakeReadCloser{ |
| {data: readBytes, counts: []int{5}, err: retryErr}, |
| {data: readBytes[5:], counts: []int{1, 3}, err: retryErr}, |
| {data: readBytes[9:], counts: []int{1}, err: io.EOF}, |
| }, |
| want: readData, |
| }, |
| { |
| offset: 0, |
| length: 5, |
| bodies: []fakeReadCloser{ |
| {data: readBytes, counts: []int{3}, err: retryErr}, |
| {data: readBytes[3:], counts: []int{2}, err: io.EOF}, |
| }, |
| want: readData[:5], |
| }, |
| { |
| offset: 1, |
| length: 5, |
| bodies: []fakeReadCloser{ |
| {data: readBytes, counts: []int{3}, err: retryErr}, |
| {data: readBytes[3:], counts: []int{2}, err: io.EOF}, |
| }, |
| want: readData[:5], |
| }, |
| { |
| offset: 1, |
| length: 3, |
| bodies: []fakeReadCloser{ |
| {data: readBytes[1:], counts: []int{1}, err: retryErr}, |
| {data: readBytes[2:], counts: []int{2}, err: io.EOF}, |
| }, |
| want: readData[1:4], |
| }, |
| { |
| offset: 4, |
| length: -1, |
| bodies: []fakeReadCloser{ |
| {data: readBytes[4:], counts: []int{1}, err: retryErr}, |
| {data: readBytes[5:], counts: []int{4}, err: retryErr}, |
| {data: readBytes[9:], counts: []int{1}, err: io.EOF}, |
| }, |
| want: readData[4:], |
| }, |
| } { |
| r, err := obj.NewRangeReader(ctx, test.offset, test.length) |
| if err != nil { |
| t.Errorf("#%d: %v", i, err) |
| continue |
| } |
| r.body = &test.bodies[0] |
| b := 0 |
| r.reopen = func(int64) (*http.Response, error) { |
| b++ |
| return &http.Response{Body: &test.bodies[b]}, nil |
| } |
| buf := make([]byte, len(readData)/2) |
| var gotb []byte |
| for { |
| n, err := r.Read(buf) |
| gotb = append(gotb, buf[:n]...) |
| if err == io.EOF { |
| break |
| } |
| if err != nil { |
| t.Fatalf("#%d: %v", i, err) |
| } |
| } |
| if err != nil { |
| t.Errorf("#%d: %v", i, err) |
| continue |
| } |
| if got := string(gotb); got != test.want { |
| t.Errorf("#%d: got %q, want %q", i, got, test.want) |
| } |
| } |
| } |
| |
| type fakeReadCloser struct { |
| data []byte |
| counts []int // how much of data to deliver on each read |
| err error // error to return with last count |
| |
| d int // current position in data |
| c int // current position in counts |
| } |
| |
| func (f *fakeReadCloser) Close() error { |
| return nil |
| } |
| |
| func (f *fakeReadCloser) Read(buf []byte) (int, error) { |
| i := f.c |
| n := 0 |
| if i < len(f.counts) { |
| n = f.counts[i] |
| } |
| var err error |
| if i >= len(f.counts)-1 { |
| err = f.err |
| } |
| copy(buf, f.data[f.d:f.d+n]) |
| if len(buf) < n { |
| n = len(buf) |
| f.counts[i] -= n |
| err = nil |
| } else { |
| f.c++ |
| } |
| f.d += n |
| return n, err |
| } |
| |
| func TestFakeReadCloser(t *testing.T) { |
| e := errors.New("") |
| f := &fakeReadCloser{ |
| data: []byte(readData), |
| counts: []int{1, 2, 3}, |
| err: e, |
| } |
| wants := []string{"0", "12", "345"} |
| buf := make([]byte, 10) |
| for i := 0; i < 3; i++ { |
| n, err := f.Read(buf) |
| if got, want := n, f.counts[i]; got != want { |
| t.Fatalf("i=%d: got %d, want %d", i, got, want) |
| } |
| var wantErr error |
| if i == 2 { |
| wantErr = e |
| } |
| if err != wantErr { |
| t.Fatalf("i=%d: got error %v, want %v", i, err, wantErr) |
| } |
| if got, want := string(buf[:n]), wants[i]; got != want { |
| t.Fatalf("i=%d: got %q, want %q", i, got, want) |
| } |
| } |
| } |