2021-10-07 14:50:36 +00:00
|
|
|
package blobstor
|
|
|
|
|
|
|
|
import (
|
2023-03-13 11:37:35 +00:00
|
|
|
"context"
|
2022-07-11 12:34:17 +00:00
|
|
|
"path/filepath"
|
2021-10-07 14:50:36 +00:00
|
|
|
"testing"
|
|
|
|
|
2023-03-07 13:38:26 +00:00
|
|
|
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
|
|
|
|
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree"
|
|
|
|
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
|
|
|
|
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree"
|
2023-03-21 10:38:44 +00:00
|
|
|
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/teststore"
|
2023-03-07 13:38:26 +00:00
|
|
|
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
2021-10-07 14:50:36 +00:00
|
|
|
"github.com/stretchr/testify/require"
|
|
|
|
)
|
|
|
|
|
2023-03-21 10:38:44 +00:00
|
|
|
func defaultTestStorages(p string, smallSizeLimit uint64) ([]SubStorage, *teststore.TestStore, *teststore.TestStore) {
|
|
|
|
smallFileStorage := teststore.New(teststore.WithSubstorage(blobovniczatree.NewBlobovniczaTree(
|
|
|
|
blobovniczatree.WithRootPath(filepath.Join(p, "blobovniczas")),
|
|
|
|
blobovniczatree.WithBlobovniczaShallowWidth(1)), // default width is 16, slow init
|
|
|
|
))
|
|
|
|
largeFileStorage := teststore.New(teststore.WithSubstorage(fstree.New(fstree.WithPath(p))))
|
2022-07-11 12:34:17 +00:00
|
|
|
return []SubStorage{
|
|
|
|
{
|
2023-03-21 10:38:44 +00:00
|
|
|
Storage: smallFileStorage,
|
2022-07-11 12:34:17 +00:00
|
|
|
Policy: func(_ *objectSDK.Object, data []byte) bool {
|
|
|
|
return uint64(len(data)) <= smallSizeLimit
|
|
|
|
},
|
|
|
|
},
|
|
|
|
{
|
2023-03-21 10:38:44 +00:00
|
|
|
Storage: largeFileStorage,
|
2022-07-11 12:34:17 +00:00
|
|
|
},
|
2023-03-21 10:38:44 +00:00
|
|
|
}, smallFileStorage, largeFileStorage
|
|
|
|
}
|
|
|
|
|
|
|
|
func defaultStorages(p string, smallSizeLimit uint64) []SubStorage {
|
|
|
|
storages, _, _ := defaultTestStorages(p, smallSizeLimit)
|
|
|
|
return storages
|
2022-07-11 12:34:17 +00:00
|
|
|
}
|
|
|
|
|
2021-10-07 14:50:36 +00:00
|
|
|
func TestCompression(t *testing.T) {
|
2023-03-21 10:38:44 +00:00
|
|
|
dir := t.TempDir()
|
2021-10-07 14:50:36 +00:00
|
|
|
|
|
|
|
const (
|
|
|
|
smallSizeLimit = 512
|
|
|
|
objCount = 4
|
|
|
|
)
|
|
|
|
|
|
|
|
newBlobStor := func(t *testing.T, compress bool) *BlobStor {
|
2022-07-11 12:34:17 +00:00
|
|
|
bs := New(
|
|
|
|
WithCompressObjects(compress),
|
|
|
|
WithStorages(defaultStorages(dir, smallSizeLimit)))
|
2022-06-28 13:42:50 +00:00
|
|
|
require.NoError(t, bs.Open(false))
|
2021-10-07 14:50:36 +00:00
|
|
|
require.NoError(t, bs.Init())
|
|
|
|
return bs
|
|
|
|
}
|
|
|
|
|
2022-03-03 14:19:05 +00:00
|
|
|
bigObj := make([]*objectSDK.Object, objCount)
|
|
|
|
smallObj := make([]*objectSDK.Object, objCount)
|
2021-10-07 14:50:36 +00:00
|
|
|
for i := 0; i < objCount; i++ {
|
|
|
|
bigObj[i] = testObject(smallSizeLimit * 2)
|
|
|
|
smallObj[i] = testObject(smallSizeLimit / 2)
|
|
|
|
}
|
|
|
|
|
|
|
|
testGet := func(t *testing.T, b *BlobStor, i int) {
|
2023-03-13 11:37:35 +00:00
|
|
|
res1, err := b.Get(context.Background(), common.GetPrm{Address: object.AddressOf(smallObj[i])})
|
2021-10-07 14:50:36 +00:00
|
|
|
require.NoError(t, err)
|
2022-07-05 14:07:40 +00:00
|
|
|
require.Equal(t, smallObj[i], res1.Object)
|
2021-10-07 14:50:36 +00:00
|
|
|
|
2023-03-13 11:37:35 +00:00
|
|
|
res2, err := b.Get(context.Background(), common.GetPrm{Address: object.AddressOf(bigObj[i])})
|
2021-10-07 14:50:36 +00:00
|
|
|
require.NoError(t, err)
|
2022-07-05 14:07:40 +00:00
|
|
|
require.Equal(t, bigObj[i], res2.Object)
|
2021-10-07 14:50:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
testPut := func(t *testing.T, b *BlobStor, i int) {
|
2022-07-06 13:41:35 +00:00
|
|
|
var prm common.PutPrm
|
|
|
|
prm.Object = smallObj[i]
|
2023-04-12 14:01:29 +00:00
|
|
|
_, err := b.Put(context.Background(), prm)
|
2021-10-07 14:50:36 +00:00
|
|
|
require.NoError(t, err)
|
|
|
|
|
2022-07-06 13:41:35 +00:00
|
|
|
prm = common.PutPrm{}
|
|
|
|
prm.Object = bigObj[i]
|
2023-04-12 14:01:29 +00:00
|
|
|
_, err = b.Put(context.Background(), prm)
|
2021-10-07 14:50:36 +00:00
|
|
|
require.NoError(t, err)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Put and Get uncompressed object
|
|
|
|
blobStor := newBlobStor(t, false)
|
|
|
|
testPut(t, blobStor, 0)
|
|
|
|
testGet(t, blobStor, 0)
|
|
|
|
require.NoError(t, blobStor.Close())
|
|
|
|
|
|
|
|
blobStor = newBlobStor(t, true)
|
|
|
|
testGet(t, blobStor, 0) // get uncompressed object with compress enabled
|
|
|
|
testPut(t, blobStor, 1)
|
|
|
|
testGet(t, blobStor, 1)
|
|
|
|
require.NoError(t, blobStor.Close())
|
|
|
|
|
|
|
|
blobStor = newBlobStor(t, false)
|
|
|
|
testGet(t, blobStor, 0) // get old uncompressed object
|
|
|
|
testGet(t, blobStor, 1) // get compressed object with compression disabled
|
|
|
|
testPut(t, blobStor, 2)
|
|
|
|
testGet(t, blobStor, 2)
|
|
|
|
require.NoError(t, blobStor.Close())
|
|
|
|
}
|
2022-01-10 12:46:01 +00:00
|
|
|
|
|
|
|
func TestBlobstor_needsCompression(t *testing.T) {
|
|
|
|
const smallSizeLimit = 512
|
|
|
|
newBlobStor := func(t *testing.T, compress bool, ct ...string) *BlobStor {
|
2023-03-21 10:38:44 +00:00
|
|
|
dir := t.TempDir()
|
2022-01-10 12:46:01 +00:00
|
|
|
|
2022-07-11 12:34:17 +00:00
|
|
|
bs := New(
|
|
|
|
WithCompressObjects(compress),
|
|
|
|
WithUncompressableContentTypes(ct),
|
|
|
|
WithStorages([]SubStorage{
|
|
|
|
{
|
|
|
|
Storage: blobovniczatree.NewBlobovniczaTree(
|
|
|
|
blobovniczatree.WithRootPath(filepath.Join(dir, "blobovnicza")),
|
|
|
|
blobovniczatree.WithBlobovniczaShallowWidth(1)), // default width is 16, slow init
|
|
|
|
Policy: func(_ *objectSDK.Object, data []byte) bool {
|
|
|
|
return uint64(len(data)) < smallSizeLimit
|
|
|
|
},
|
|
|
|
},
|
|
|
|
{
|
|
|
|
Storage: fstree.New(fstree.WithPath(dir)),
|
|
|
|
},
|
|
|
|
}))
|
2022-06-28 13:42:50 +00:00
|
|
|
require.NoError(t, bs.Open(false))
|
2022-01-10 12:46:01 +00:00
|
|
|
require.NoError(t, bs.Init())
|
|
|
|
return bs
|
|
|
|
}
|
|
|
|
|
2022-03-03 14:19:05 +00:00
|
|
|
newObjectWithCt := func(contentType string) *objectSDK.Object {
|
|
|
|
obj := testObject(smallSizeLimit + 1)
|
2022-01-10 12:46:01 +00:00
|
|
|
if contentType != "" {
|
2022-03-15 12:11:35 +00:00
|
|
|
var a objectSDK.Attribute
|
2022-01-10 12:46:01 +00:00
|
|
|
a.SetKey(objectSDK.AttributeContentType)
|
|
|
|
a.SetValue(contentType)
|
|
|
|
obj.SetAttributes(a)
|
|
|
|
}
|
2022-03-03 14:19:05 +00:00
|
|
|
return obj
|
2022-01-10 12:46:01 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
t.Run("content-types specified", func(t *testing.T) {
|
|
|
|
b := newBlobStor(t, true, "audio/*", "*/x-mpeg", "*/mpeg", "application/x-midi")
|
|
|
|
|
|
|
|
obj := newObjectWithCt("video/mpeg")
|
2022-01-11 11:33:04 +00:00
|
|
|
require.False(t, b.NeedsCompression(obj))
|
2022-01-10 12:46:01 +00:00
|
|
|
|
|
|
|
obj = newObjectWithCt("audio/aiff")
|
2022-01-11 11:33:04 +00:00
|
|
|
require.False(t, b.NeedsCompression(obj))
|
2022-01-10 12:46:01 +00:00
|
|
|
|
|
|
|
obj = newObjectWithCt("application/x-midi")
|
2022-01-11 11:33:04 +00:00
|
|
|
require.False(t, b.NeedsCompression(obj))
|
2022-01-10 12:46:01 +00:00
|
|
|
|
|
|
|
obj = newObjectWithCt("text/plain")
|
2022-01-11 11:33:04 +00:00
|
|
|
require.True(t, b.NeedsCompression(obj))
|
2022-01-10 12:46:01 +00:00
|
|
|
|
|
|
|
obj = newObjectWithCt("")
|
2022-01-11 11:33:04 +00:00
|
|
|
require.True(t, b.NeedsCompression(obj))
|
2022-01-10 12:46:01 +00:00
|
|
|
})
|
|
|
|
t.Run("content-types omitted", func(t *testing.T) {
|
|
|
|
b := newBlobStor(t, true)
|
|
|
|
obj := newObjectWithCt("video/mpeg")
|
2022-01-11 11:33:04 +00:00
|
|
|
require.True(t, b.NeedsCompression(obj))
|
2022-01-10 12:46:01 +00:00
|
|
|
})
|
|
|
|
t.Run("compress disabled", func(t *testing.T) {
|
|
|
|
b := newBlobStor(t, false, "video/mpeg")
|
|
|
|
|
|
|
|
obj := newObjectWithCt("video/mpeg")
|
2022-01-11 11:33:04 +00:00
|
|
|
require.False(t, b.NeedsCompression(obj))
|
2022-01-10 12:46:01 +00:00
|
|
|
|
|
|
|
obj = newObjectWithCt("text/plain")
|
2022-01-11 11:33:04 +00:00
|
|
|
require.False(t, b.NeedsCompression(obj))
|
2022-01-10 12:46:01 +00:00
|
|
|
})
|
|
|
|
}
|