blob: 3b3252b7e7852b16f9c3460d684a13f126775caf [file] [log] [blame]
// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package httpreplay_test
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"testing"
"time"
"cloud.google.com/go/httpreplay"
"cloud.google.com/go/internal/testutil"
"cloud.google.com/go/storage"
"google.golang.org/api/option"
)
const (
compressedFile = "httpreplay_compressed.txt"
uncompressedFile = "httpreplay_uncompressed.txt"
)
func TestIntegration_RecordAndReplay(t *testing.T) {
httpreplay.DebugHeaders()
if testing.Short() {
t.Skip("Integration tests skipped in short mode")
}
replayFilename := tempFilename(t, "RecordAndReplay*.replay")
defer os.Remove(replayFilename)
projectID := testutil.ProjID()
if projectID == "" {
t.Skip("Need project ID. See CONTRIBUTING.md for details.")
}
ctx := context.Background()
cleanup, err := setup(ctx)
if err != nil {
t.Fatal(err)
}
defer cleanup()
// Record.
initial := time.Now()
ibytes, err := json.Marshal(initial)
if err != nil {
t.Fatal(err)
}
rec, err := httpreplay.NewRecorder(replayFilename, ibytes)
if err != nil {
t.Fatal(err)
}
hc, err := rec.Client(ctx, option.WithTokenSource(
testutil.TokenSource(ctx, storage.ScopeFullControl)))
if err != nil {
t.Fatal(err)
}
wanta, wantc := run(t, hc)
testReadCRC(t, hc, "recording")
if err := rec.Close(); err != nil {
t.Fatalf("rec.Close: %v", err)
}
// Replay.
rep, err := httpreplay.NewReplayer(replayFilename)
if err != nil {
t.Fatal(err)
}
defer rep.Close()
hc, err = rep.Client(ctx)
if err != nil {
t.Fatal(err)
}
gota, gotc := run(t, hc)
testReadCRC(t, hc, "replaying")
if diff := testutil.Diff(gota, wanta); diff != "" {
t.Error(diff)
}
if !bytes.Equal(gotc, wantc) {
t.Errorf("got %q, want %q", gotc, wantc)
}
var gotInitial time.Time
if err := json.Unmarshal(rep.Initial(), &gotInitial); err != nil {
t.Fatal(err)
}
if !gotInitial.Equal(initial) {
t.Errorf("initial: got %v, want %v", gotInitial, initial)
}
}
func setup(ctx context.Context) (cleanup func(), err error) {
ts := testutil.TokenSource(ctx, storage.ScopeFullControl)
client, err := storage.NewClient(ctx, option.WithTokenSource(ts))
if err != nil {
return nil, err
}
bucket := testutil.ProjID()
// upload compressed object
f1, err := os.Open(filepath.Join("internal", "testdata", "compressed.txt"))
if err != nil {
return nil, err
}
defer f1.Close()
w := client.Bucket(bucket).Object(compressedFile).NewWriter(ctx)
w.ContentEncoding = "gzip"
w.ContentType = "text/plain"
if _, err = io.Copy(w, f1); err != nil {
return nil, err
}
if err := w.Close(); err != nil {
return nil, err
}
// upload uncompressed object
f2, err := os.Open(filepath.Join("internal", "testdata", "uncompressed.txt"))
if err != nil {
return nil, err
}
defer f2.Close()
w = client.Bucket(testutil.ProjID()).Object(uncompressedFile).NewWriter(ctx)
if _, err = io.Copy(w, f2); err != nil {
return nil, err
}
if err := w.Close(); err != nil {
return nil, err
}
return func() {
client.Bucket(bucket).Object(compressedFile).Delete(ctx)
client.Bucket(bucket).Object(uncompressedFile).Delete(ctx)
client.Close()
}, nil
}
// TODO(jba): test errors
func run(t *testing.T, hc *http.Client) (*storage.BucketAttrs, []byte) {
ctx := context.Background()
client, err := storage.NewClient(ctx, option.WithHTTPClient(hc))
if err != nil {
t.Fatal(err)
}
defer client.Close()
b := client.Bucket(testutil.ProjID())
attrs, err := b.Attrs(ctx)
if err != nil {
t.Fatal(err)
}
obj := b.Object("replay-test")
w := obj.NewWriter(ctx)
data := []byte{150, 151, 152}
if _, err := w.Write(data); err != nil {
t.Fatal(err)
}
if err := w.Close(); err != nil {
t.Fatal(err)
}
r, err := obj.NewReader(ctx)
if err != nil {
t.Fatal(err)
}
defer r.Close()
contents, err := ioutil.ReadAll(r)
if err != nil {
t.Fatal(err)
}
return attrs, contents
}
func testReadCRC(t *testing.T, hc *http.Client, mode string) {
ctx := context.Background()
client, err := storage.NewClient(ctx, option.WithHTTPClient(hc))
if err != nil {
t.Fatalf("%s: %v", mode, err)
}
defer client.Close()
bucket := testutil.ProjID()
uncompressedObj := client.Bucket(bucket).Object(uncompressedFile)
gzippedObj := client.Bucket(bucket).Object(compressedFile)
for _, test := range []struct {
desc string
obj *storage.ObjectHandle
offset, length int64
readCompressed bool // don't decompress a gzipped file
wantErr bool
wantLen int // length of contents
}{
{
desc: "uncompressed, entire file",
obj: uncompressedObj,
offset: 0,
length: -1,
readCompressed: false,
wantLen: 179,
},
{
desc: "uncompressed, entire file, don't decompress",
obj: uncompressedObj,
offset: 0,
length: -1,
readCompressed: true,
wantLen: 179,
},
{
desc: "uncompressed, suffix",
obj: uncompressedObj,
offset: 9,
length: -1,
readCompressed: false,
wantLen: 170,
},
{
desc: "uncompressed, prefix",
obj: uncompressedObj,
offset: 0,
length: 18,
readCompressed: false,
wantLen: 18,
},
{
// When a gzipped file is unzipped by GCS, we can't verify the checksum
// because it was computed against the zipped contents. There is no
// header that indicates that a gzipped file is being served unzipped.
// But our CRC check only happens if there is a Content-Length header,
// and that header is absent for this read.
desc: "compressed, entire file, server unzips",
obj: gzippedObj,
offset: 0,
length: -1,
readCompressed: false,
wantLen: 179,
},
{
// When we read a gzipped file uncompressed, it's like reading a regular file:
// the served content and the CRC match.
desc: "compressed, entire file, read compressed",
obj: gzippedObj,
offset: 0,
length: -1,
readCompressed: true,
wantLen: 128,
},
{
desc: "compressed, partial, read compressed",
obj: gzippedObj,
offset: 1,
length: 8,
readCompressed: true,
wantLen: 8,
},
{
desc: "uncompressed, HEAD",
obj: uncompressedObj,
offset: 0,
length: 0,
wantLen: 0,
},
{
desc: "compressed, HEAD",
obj: gzippedObj,
offset: 0,
length: 0,
wantLen: 0,
},
} {
obj := test.obj.ReadCompressed(test.readCompressed)
r, err := obj.NewRangeReader(ctx, test.offset, test.length)
if err != nil {
if test.wantErr {
continue
}
t.Errorf("%s: %s: %v", mode, test.desc, err)
continue
}
data, err := ioutil.ReadAll(r)
_ = r.Close()
if err != nil {
t.Errorf("%s: %s: %v", mode, test.desc, err)
continue
}
if got, want := len(data), test.wantLen; got != want {
t.Errorf("%s: %s: len: got %d, want %d", mode, test.desc, got, want)
}
}
}
func TestRemoveAndClear(t *testing.T) {
// Disable logging for this test, since it generates a lot.
log.SetOutput(ioutil.Discard)
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
fmt.Fprintln(w, "LGTM")
}))
defer srv.Close()
replayFilename := tempFilename(t, "TestRemoveAndClear*.replay")
defer os.Remove(replayFilename)
ctx := context.Background()
// Record
rec, err := httpreplay.NewRecorder(replayFilename, nil)
if err != nil {
t.Fatal(err)
}
rec.ClearHeaders("Clear")
rec.RemoveRequestHeaders("Rem*")
rec.ClearQueryParams("c")
rec.RemoveQueryParams("r")
hc, err := rec.Client(ctx, option.WithoutAuthentication())
if err != nil {
t.Fatal(err)
}
query := "k=1&r=2&c=3"
req, err := http.NewRequest("GET", srv.URL+"?"+query, nil)
if err != nil {
t.Fatal(err)
}
headers := map[string]string{"Keep": "ok", "Clear": "secret", "Remove": "bye"}
for k, v := range headers {
req.Header.Set(k, v)
}
if _, err := hc.Do(req); err != nil {
t.Fatal(err)
}
if err := rec.Close(); err != nil {
t.Fatal(err)
}
// Replay
// For both headers and query param:
// - k or Keep must be present and identical
// - c or Clear must be present, but can be different
// - r or Remove can be anything
for _, test := range []struct {
query string
headers map[string]string
wantSuccess bool
}{
{query, headers, true}, // same query string and headers
{query,
map[string]string{"Keep": "oops", "Clear": "secret", "Remove": "bye"},
false, // different Keep
},
{query, map[string]string{}, false}, // missing Keep and Clear
{query, map[string]string{"Keep": "ok"}, false}, // missing Clear
{query, map[string]string{"Keep": "ok", "Clear": "secret"}, true}, // missing Remove is OK
{
query,
map[string]string{"Keep": "ok", "Clear": "secret", "Remove": "whatev"},
true,
}, // different Remove is OK
{query, map[string]string{"Keep": "ok", "Clear": "diff"}, true}, // different Clear is OK
{"", headers, false}, // no query string
{"k=x&r=2&c=3", headers, false}, // different k
{"r=2", headers, false}, // missing k and c
{"k=1&r=2", headers, false}, // missing c
{"k=1&c=3", headers, true}, // missing r is OK
{"k=1&r=x&c=3", headers, true}, // different r is OK,
{"k=1&r=2&c=x", headers, true}, // different clear is OK
} {
rep, err := httpreplay.NewReplayer(replayFilename)
if err != nil {
t.Fatal(err)
}
hc, err = rep.Client(ctx)
if err != nil {
t.Fatal(err)
}
url := srv.URL
if test.query != "" {
url += "?" + test.query
}
req, err = http.NewRequest("GET", url, nil)
if err != nil {
t.Fatal(err)
}
for k, v := range test.headers {
req.Header.Set(k, v)
}
resp, err := hc.Do(req)
if err != nil {
t.Fatal(err)
}
rep.Close()
if (resp.StatusCode == 200) != test.wantSuccess {
t.Errorf("%q, %v: got %d, wanted success=%t",
test.query, test.headers, resp.StatusCode, test.wantSuccess)
}
}
}
func tempFilename(t *testing.T, pattern string) string {
f, err := ioutil.TempFile("", pattern)
if err != nil {
t.Fatal(err)
}
filename := f.Name()
if err := f.Close(); err != nil {
t.Fatal(err)
}
return filename
}