5c372ded1b
This change refreshes the updated version of Azure SDK for Go that has the latest changes. I manually vendored the new SDK (github.com/Azure/azure-sdk-for-go) and I removed `management/` `core/` packages manually simply because they're not used here and they have a fork of `net/http` and `crypto/tls` for a particular reason. It was introducing a 44k SLOC change otherwise... This also undoes the `include_azure` flag (actually Steven removed the driver from imports but forgot to add the build flag apparently, so the flag wasn't really including azure. 😄 ). This also must be obsolete now. Fixes #620, #175. Signed-off-by: Ahmet Alp Balkan <ahmetalpbalkan@gmail.com>
339 lines
11 KiB
Go
339 lines
11 KiB
Go
package azure
|
|
|
|
import (
|
|
"bytes"
|
|
"io"
|
|
"io/ioutil"
|
|
"math/rand"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
|
|
azure "github.com/Azure/azure-sdk-for-go/storage"
|
|
)
|
|
|
|
func TestRandomWriter_writeChunkToBlocks(t *testing.T) {
|
|
s := NewStorageSimulator()
|
|
rw := newRandomBlobWriter(&s, 3)
|
|
rand := newBlockIDGenerator()
|
|
c := []byte("AAABBBCCCD")
|
|
|
|
if err := rw.bs.CreateBlockBlob("a", "b"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
bw, nn, err := rw.writeChunkToBlocks("a", "b", bytes.NewReader(c), rand)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if expected := int64(len(c)); nn != expected {
|
|
t.Fatalf("wrong nn:%v, expected:%v", nn, expected)
|
|
}
|
|
if expected := 4; len(bw) != expected {
|
|
t.Fatal("unexpected written block count")
|
|
}
|
|
|
|
bx, err := s.GetBlockList("a", "b", azure.BlockListTypeAll)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if expected := 0; len(bx.CommittedBlocks) != expected {
|
|
t.Fatal("unexpected committed block count")
|
|
}
|
|
if expected := 4; len(bx.UncommittedBlocks) != expected {
|
|
t.Fatalf("unexpected uncommitted block count: %d -- %#v", len(bx.UncommittedBlocks), bx)
|
|
}
|
|
|
|
if err := rw.bs.PutBlockList("a", "b", bw); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
r, err := rw.bs.GetBlob("a", "b")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
assertBlobContents(t, r, c)
|
|
}
|
|
|
|
func TestRandomWriter_blocksLeftSide(t *testing.T) {
|
|
blob := "AAAAABBBBBCCC"
|
|
cases := []struct {
|
|
offset int64
|
|
expectedBlob string
|
|
expectedPattern []azure.BlockStatus
|
|
}{
|
|
{0, "", []azure.BlockStatus{}}, // write to beginning, discard all
|
|
{13, blob, []azure.BlockStatus{azure.BlockStatusCommitted, azure.BlockStatusCommitted, azure.BlockStatusCommitted}}, // write to end, no change
|
|
{1, "A", []azure.BlockStatus{azure.BlockStatusUncommitted}}, // write at 1
|
|
{5, "AAAAA", []azure.BlockStatus{azure.BlockStatusCommitted}}, // write just after first block
|
|
{6, "AAAAAB", []azure.BlockStatus{azure.BlockStatusCommitted, azure.BlockStatusUncommitted}}, // split the second block
|
|
{9, "AAAAABBBB", []azure.BlockStatus{azure.BlockStatusCommitted, azure.BlockStatusUncommitted}}, // write just after first block
|
|
}
|
|
|
|
for _, c := range cases {
|
|
s := NewStorageSimulator()
|
|
rw := newRandomBlobWriter(&s, 5)
|
|
rand := newBlockIDGenerator()
|
|
|
|
if err := rw.bs.CreateBlockBlob("a", "b"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
bw, _, err := rw.writeChunkToBlocks("a", "b", strings.NewReader(blob), rand)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := rw.bs.PutBlockList("a", "b", bw); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
bx, err := rw.blocksLeftSide("a", "b", c.offset, rand)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
bs := []azure.BlockStatus{}
|
|
for _, v := range bx {
|
|
bs = append(bs, v.Status)
|
|
}
|
|
|
|
if !reflect.DeepEqual(bs, c.expectedPattern) {
|
|
t.Logf("Committed blocks %v", bw)
|
|
t.Fatalf("For offset %v: Expected pattern: %v, Got: %v\n(Returned: %v)", c.offset, c.expectedPattern, bs, bx)
|
|
}
|
|
if rw.bs.PutBlockList("a", "b", bx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r, err := rw.bs.GetBlob("a", "b")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
cout, err := ioutil.ReadAll(r)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
outBlob := string(cout)
|
|
if outBlob != c.expectedBlob {
|
|
t.Fatalf("wrong blob contents: %v, expected: %v", outBlob, c.expectedBlob)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRandomWriter_blocksRightSide(t *testing.T) {
|
|
blob := "AAAAABBBBBCCC"
|
|
cases := []struct {
|
|
offset int64
|
|
size int64
|
|
expectedBlob string
|
|
expectedPattern []azure.BlockStatus
|
|
}{
|
|
{0, 100, "", []azure.BlockStatus{}}, // overwrite the entire blob
|
|
{0, 3, "AABBBBBCCC", []azure.BlockStatus{azure.BlockStatusUncommitted, azure.BlockStatusCommitted, azure.BlockStatusCommitted}}, // split first block
|
|
{4, 1, "BBBBBCCC", []azure.BlockStatus{azure.BlockStatusCommitted, azure.BlockStatusCommitted}}, // write to last char of first block
|
|
{1, 6, "BBBCCC", []azure.BlockStatus{azure.BlockStatusUncommitted, azure.BlockStatusCommitted}}, // overwrite splits first and second block, last block remains
|
|
{3, 8, "CC", []azure.BlockStatus{azure.BlockStatusUncommitted}}, // overwrite a block in middle block, split end block
|
|
{10, 1, "CC", []azure.BlockStatus{azure.BlockStatusUncommitted}}, // overwrite first byte of rightmost block
|
|
{11, 2, "", []azure.BlockStatus{}}, // overwrite the rightmost index
|
|
{13, 20, "", []azure.BlockStatus{}}, // append to the end
|
|
}
|
|
|
|
for _, c := range cases {
|
|
s := NewStorageSimulator()
|
|
rw := newRandomBlobWriter(&s, 5)
|
|
rand := newBlockIDGenerator()
|
|
|
|
if err := rw.bs.CreateBlockBlob("a", "b"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
bw, _, err := rw.writeChunkToBlocks("a", "b", strings.NewReader(blob), rand)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := rw.bs.PutBlockList("a", "b", bw); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
bx, err := rw.blocksRightSide("a", "b", c.offset, c.size, rand)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
bs := []azure.BlockStatus{}
|
|
for _, v := range bx {
|
|
bs = append(bs, v.Status)
|
|
}
|
|
|
|
if !reflect.DeepEqual(bs, c.expectedPattern) {
|
|
t.Logf("Committed blocks %v", bw)
|
|
t.Fatalf("For offset %v-size:%v: Expected pattern: %v, Got: %v\n(Returned: %v)", c.offset, c.size, c.expectedPattern, bs, bx)
|
|
}
|
|
if rw.bs.PutBlockList("a", "b", bx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r, err := rw.bs.GetBlob("a", "b")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
cout, err := ioutil.ReadAll(r)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
outBlob := string(cout)
|
|
if outBlob != c.expectedBlob {
|
|
t.Fatalf("For offset %v-size:%v: wrong blob contents: %v, expected: %v", c.offset, c.size, outBlob, c.expectedBlob)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRandomWriter_Write_NewBlob(t *testing.T) {
|
|
var (
|
|
s = NewStorageSimulator()
|
|
rw = newRandomBlobWriter(&s, 1024*3) // 3 KB blocks
|
|
blob = randomContents(1024 * 7) // 7 KB blob
|
|
)
|
|
if err := rw.bs.CreateBlockBlob("a", "b"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
if _, err := rw.WriteBlobAt("a", "b", 10, bytes.NewReader(blob)); err == nil {
|
|
t.Fatal("expected error, got nil")
|
|
}
|
|
if _, err := rw.WriteBlobAt("a", "b", 100000, bytes.NewReader(blob)); err == nil {
|
|
t.Fatal("expected error, got nil")
|
|
}
|
|
if nn, err := rw.WriteBlobAt("a", "b", 0, bytes.NewReader(blob)); err != nil {
|
|
t.Fatal(err)
|
|
} else if expected := int64(len(blob)); expected != nn {
|
|
t.Fatalf("wrong written bytes count: %v, expected: %v", nn, expected)
|
|
}
|
|
if out, err := rw.bs.GetBlob("a", "b"); err != nil {
|
|
t.Fatal(err)
|
|
} else {
|
|
assertBlobContents(t, out, blob)
|
|
}
|
|
if bx, err := rw.bs.GetBlockList("a", "b", azure.BlockListTypeCommitted); err != nil {
|
|
t.Fatal(err)
|
|
} else if len(bx.CommittedBlocks) != 3 {
|
|
t.Fatalf("got wrong number of committed blocks: %v", len(bx.CommittedBlocks))
|
|
}
|
|
|
|
// Replace first 512 bytes
|
|
leftChunk := randomContents(512)
|
|
blob = append(leftChunk, blob[512:]...)
|
|
if nn, err := rw.WriteBlobAt("a", "b", 0, bytes.NewReader(leftChunk)); err != nil {
|
|
t.Fatal(err)
|
|
} else if expected := int64(len(leftChunk)); expected != nn {
|
|
t.Fatalf("wrong written bytes count: %v, expected: %v", nn, expected)
|
|
}
|
|
if out, err := rw.bs.GetBlob("a", "b"); err != nil {
|
|
t.Fatal(err)
|
|
} else {
|
|
assertBlobContents(t, out, blob)
|
|
}
|
|
if bx, err := rw.bs.GetBlockList("a", "b", azure.BlockListTypeCommitted); err != nil {
|
|
t.Fatal(err)
|
|
} else if expected := 4; len(bx.CommittedBlocks) != expected {
|
|
t.Fatalf("got wrong number of committed blocks: %v, expected: %v", len(bx.CommittedBlocks), expected)
|
|
}
|
|
|
|
// Replace last 512 bytes with 1024 bytes
|
|
rightChunk := randomContents(1024)
|
|
offset := int64(len(blob) - 512)
|
|
blob = append(blob[:offset], rightChunk...)
|
|
if nn, err := rw.WriteBlobAt("a", "b", offset, bytes.NewReader(rightChunk)); err != nil {
|
|
t.Fatal(err)
|
|
} else if expected := int64(len(rightChunk)); expected != nn {
|
|
t.Fatalf("wrong written bytes count: %v, expected: %v", nn, expected)
|
|
}
|
|
if out, err := rw.bs.GetBlob("a", "b"); err != nil {
|
|
t.Fatal(err)
|
|
} else {
|
|
assertBlobContents(t, out, blob)
|
|
}
|
|
if bx, err := rw.bs.GetBlockList("a", "b", azure.BlockListTypeCommitted); err != nil {
|
|
t.Fatal(err)
|
|
} else if expected := 5; len(bx.CommittedBlocks) != expected {
|
|
t.Fatalf("got wrong number of committed blocks: %v, expected: %v", len(bx.CommittedBlocks), expected)
|
|
}
|
|
|
|
// Replace 2K-4K (overlaps 2 blocks from L/R)
|
|
newChunk := randomContents(1024 * 2)
|
|
offset = 1024 * 2
|
|
blob = append(append(blob[:offset], newChunk...), blob[offset+int64(len(newChunk)):]...)
|
|
if nn, err := rw.WriteBlobAt("a", "b", offset, bytes.NewReader(newChunk)); err != nil {
|
|
t.Fatal(err)
|
|
} else if expected := int64(len(newChunk)); expected != nn {
|
|
t.Fatalf("wrong written bytes count: %v, expected: %v", nn, expected)
|
|
}
|
|
if out, err := rw.bs.GetBlob("a", "b"); err != nil {
|
|
t.Fatal(err)
|
|
} else {
|
|
assertBlobContents(t, out, blob)
|
|
}
|
|
if bx, err := rw.bs.GetBlockList("a", "b", azure.BlockListTypeCommitted); err != nil {
|
|
t.Fatal(err)
|
|
} else if expected := 6; len(bx.CommittedBlocks) != expected {
|
|
t.Fatalf("got wrong number of committed blocks: %v, expected: %v\n%v", len(bx.CommittedBlocks), expected, bx.CommittedBlocks)
|
|
}
|
|
|
|
// Replace the entire blob
|
|
newBlob := randomContents(1024 * 30)
|
|
if nn, err := rw.WriteBlobAt("a", "b", 0, bytes.NewReader(newBlob)); err != nil {
|
|
t.Fatal(err)
|
|
} else if expected := int64(len(newBlob)); expected != nn {
|
|
t.Fatalf("wrong written bytes count: %v, expected: %v", nn, expected)
|
|
}
|
|
if out, err := rw.bs.GetBlob("a", "b"); err != nil {
|
|
t.Fatal(err)
|
|
} else {
|
|
assertBlobContents(t, out, newBlob)
|
|
}
|
|
if bx, err := rw.bs.GetBlockList("a", "b", azure.BlockListTypeCommitted); err != nil {
|
|
t.Fatal(err)
|
|
} else if expected := 10; len(bx.CommittedBlocks) != expected {
|
|
t.Fatalf("got wrong number of committed blocks: %v, expected: %v\n%v", len(bx.CommittedBlocks), expected, bx.CommittedBlocks)
|
|
} else if expected, size := int64(1024*30), getBlobSize(bx); size != expected {
|
|
t.Fatalf("committed block size does not indicate blob size")
|
|
}
|
|
}
|
|
|
|
func Test_getBlobSize(t *testing.T) {
|
|
// with some committed blocks
|
|
if expected, size := int64(151), getBlobSize(azure.BlockListResponse{
|
|
CommittedBlocks: []azure.BlockResponse{
|
|
{"A", 100},
|
|
{"B", 50},
|
|
{"C", 1},
|
|
},
|
|
UncommittedBlocks: []azure.BlockResponse{
|
|
{"D", 200},
|
|
}}); expected != size {
|
|
t.Fatalf("wrong blob size: %v, expected: %v", size, expected)
|
|
}
|
|
|
|
// with no committed blocks
|
|
if expected, size := int64(0), getBlobSize(azure.BlockListResponse{
|
|
UncommittedBlocks: []azure.BlockResponse{
|
|
{"A", 100},
|
|
{"B", 50},
|
|
{"C", 1},
|
|
{"D", 200},
|
|
}}); expected != size {
|
|
t.Fatalf("wrong blob size: %v, expected: %v", size, expected)
|
|
}
|
|
}
|
|
|
|
func assertBlobContents(t *testing.T, r io.Reader, expected []byte) {
|
|
out, err := ioutil.ReadAll(r)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
if !reflect.DeepEqual(out, expected) {
|
|
t.Fatalf("wrong blob contents. size: %v, expected: %v", len(out), len(expected))
|
|
}
|
|
}
|
|
|
|
func randomContents(length int64) []byte {
|
|
b := make([]byte, length)
|
|
for i := range b {
|
|
b[i] = byte(rand.Intn(2 << 8))
|
|
}
|
|
return b
|
|
}
|