frostfs-s3-gw/api/layer/versioning_test.go
Denis Kirillov 29ac91dfd5 [#165] Support streaming listing
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2024-02-02 16:09:48 +03:00

445 lines
13 KiB
Go

package layer
import (
"bytes"
"context"
"io"
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
bearertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer/test"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
"github.com/stretchr/testify/require"
"go.uber.org/zap"
)
func (tc *testContext) putObject(content []byte) *data.ObjectInfo {
extObjInfo, err := tc.layer.PutObject(tc.ctx, &PutObjectParams{
BktInfo: tc.bktInfo,
Object: tc.obj,
Size: uint64(len(content)),
Reader: bytes.NewReader(content),
Header: make(map[string]string),
})
require.NoError(tc.t, err)
return extObjInfo.ObjectInfo
}
func (tc *testContext) getObject(objectName, versionID string, needError bool) (*data.ObjectInfo, []byte) {
headPrm := &HeadObjectParams{
BktInfo: tc.bktInfo,
Object: objectName,
VersionID: versionID,
}
objInfo, err := tc.layer.GetObjectInfo(tc.ctx, headPrm)
if needError {
require.Error(tc.t, err)
return nil, nil
}
require.NoError(tc.t, err)
objPayload, err := tc.layer.GetObject(tc.ctx, &GetObjectParams{
ObjectInfo: objInfo,
Versioned: headPrm.Versioned(),
BucketInfo: tc.bktInfo,
})
require.NoError(tc.t, err)
payload, err := io.ReadAll(objPayload)
require.NoError(tc.t, err)
return objInfo, payload
}
func (tc *testContext) deleteObject(objectName, versionID string, settings *data.BucketSettings) {
p := &DeleteObjectParams{
BktInfo: tc.bktInfo,
Settings: settings,
Objects: []*VersionedObject{
{Name: objectName, VersionID: versionID},
},
}
deletedObjects := tc.layer.DeleteObjects(tc.ctx, p)
for _, obj := range deletedObjects {
require.NoError(tc.t, obj.Error)
}
}
func (tc *testContext) listObjectsV1() []*data.ObjectInfo {
res, err := tc.layer.ListObjectsV1(tc.ctx, &ListObjectsParamsV1{
ListObjectsParamsCommon: ListObjectsParamsCommon{
BktInfo: tc.bktInfo,
MaxKeys: 1000,
},
})
require.NoError(tc.t, err)
return res.Objects
}
func (tc *testContext) listObjectsV2() []*data.ObjectInfo {
res, err := tc.layer.ListObjectsV2(tc.ctx, &ListObjectsParamsV2{
ListObjectsParamsCommon: ListObjectsParamsCommon{
BktInfo: tc.bktInfo,
MaxKeys: 1000,
},
})
require.NoError(tc.t, err)
return res.Objects
}
func (tc *testContext) listVersions() *ListObjectVersionsInfo {
res, err := tc.layer.ListObjectVersions(tc.ctx, &ListObjectVersionsParams{
BktInfo: tc.bktInfo,
MaxKeys: 1000,
})
require.NoError(tc.t, err)
return res
}
func (tc *testContext) checkListObjects(ids ...oid.ID) {
objs := tc.listObjectsV1()
require.Equal(tc.t, len(ids), len(objs))
for _, id := range ids {
require.Contains(tc.t, ids, id)
}
objs = tc.listObjectsV2()
require.Equal(tc.t, len(ids), len(objs))
for _, id := range ids {
require.Contains(tc.t, ids, id)
}
}
func (tc *testContext) getObjectByID(objID oid.ID) *object.Object {
for _, obj := range tc.testFrostFS.Objects() {
id, _ := obj.ID()
if id.Equals(objID) {
return obj
}
}
return nil
}
type testContext struct {
t *testing.T
ctx context.Context
layer Client
bktInfo *data.BucketInfo
obj string
testFrostFS *TestFrostFS
}
func prepareContext(t *testing.T, cachesConfig ...*CachesConfig) *testContext {
logger := zap.NewExample()
key, err := keys.NewPrivateKey()
require.NoError(t, err)
bearerToken := bearertest.Token()
require.NoError(t, bearerToken.Sign(key.PrivateKey))
ctx := middleware.SetBoxData(context.Background(), &accessbox.Box{
Gate: &accessbox.GateData{
BearerToken: &bearerToken,
GateKey: key.PublicKey(),
},
})
tp := NewTestFrostFS(key)
bktName := "testbucket1"
res, err := tp.CreateContainer(ctx, PrmContainerCreate{
Name: bktName,
})
require.NoError(t, err)
config := DefaultCachesConfigs(logger)
if len(cachesConfig) != 0 {
config = cachesConfig[0]
}
var owner user.ID
user.IDFromKey(&owner, key.PrivateKey.PublicKey)
layerCfg := &Config{
Caches: config,
AnonKey: AnonymousKey{Key: key},
TreeService: NewTreeService(),
Features: &FeatureSettingsMock{},
}
return &testContext{
ctx: ctx,
layer: NewLayer(logger, tp, layerCfg),
bktInfo: &data.BucketInfo{
Name: bktName,
Owner: owner,
CID: res.ContainerID,
HomomorphicHashDisabled: res.HomomorphicHashDisabled,
},
obj: "obj1",
t: t,
testFrostFS: tp,
}
}
func TestSimpleVersioning(t *testing.T) {
tc := prepareContext(t)
err := tc.layer.PutBucketSettings(tc.ctx, &PutSettingsParams{
BktInfo: tc.bktInfo,
Settings: &data.BucketSettings{Versioning: data.VersioningEnabled},
})
require.NoError(t, err)
obj1Content1 := []byte("content obj1 v1")
obj1v1 := tc.putObject(obj1Content1)
obj1Content2 := []byte("content obj1 v2")
obj1v2 := tc.putObject(obj1Content2)
_, buffer2 := tc.getObject(tc.obj, "", false)
require.Equal(t, obj1Content2, buffer2)
_, buffer1 := tc.getObject(tc.obj, obj1v1.ID.EncodeToString(), false)
require.Equal(t, obj1Content1, buffer1)
tc.checkListObjects(obj1v2.ID)
}
func TestSimpleNoVersioning(t *testing.T) {
tc := prepareContext(t)
obj1Content1 := []byte("content obj1 v1")
obj1v1 := tc.putObject(obj1Content1)
obj1Content2 := []byte("content obj1 v2")
obj1v2 := tc.putObject(obj1Content2)
_, buffer2 := tc.getObject(tc.obj, "", false)
require.Equal(t, obj1Content2, buffer2)
tc.getObject(tc.obj, obj1v1.ID.EncodeToString(), true)
tc.checkListObjects(obj1v2.ID)
}
func TestVersioningDeleteObject(t *testing.T) {
tc := prepareContext(t)
settings := &data.BucketSettings{Versioning: data.VersioningEnabled}
err := tc.layer.PutBucketSettings(tc.ctx, &PutSettingsParams{
BktInfo: tc.bktInfo,
Settings: settings,
})
require.NoError(t, err)
tc.putObject([]byte("content obj1 v1"))
tc.putObject([]byte("content obj1 v2"))
tc.deleteObject(tc.obj, "", settings)
tc.getObject(tc.obj, "", true)
tc.checkListObjects()
}
func TestGetUnversioned(t *testing.T) {
tc := prepareContext(t)
objContent := []byte("content obj1 v1")
objInfo := tc.putObject(objContent)
settings := &data.BucketSettings{Versioning: data.VersioningUnversioned}
err := tc.layer.PutBucketSettings(tc.ctx, &PutSettingsParams{
BktInfo: tc.bktInfo,
Settings: settings,
})
require.NoError(t, err)
resInfo, buffer := tc.getObject(tc.obj, data.UnversionedObjectVersionID, false)
require.Equal(t, objContent, buffer)
require.Equal(t, objInfo.VersionID(), resInfo.VersionID())
}
func TestVersioningDeleteSpecificObjectVersion(t *testing.T) {
tc := prepareContext(t)
settings := &data.BucketSettings{Versioning: data.VersioningEnabled}
err := tc.layer.PutBucketSettings(tc.ctx, &PutSettingsParams{
BktInfo: tc.bktInfo,
Settings: settings,
})
require.NoError(t, err)
tc.putObject([]byte("content obj1 v1"))
objV2Info := tc.putObject([]byte("content obj1 v2"))
objV3Content := []byte("content obj1 v3")
objV3Info := tc.putObject(objV3Content)
tc.deleteObject(tc.obj, objV2Info.VersionID(), settings)
tc.getObject(tc.obj, objV2Info.VersionID(), true)
_, buffer3 := tc.getObject(tc.obj, "", false)
require.Equal(t, objV3Content, buffer3)
tc.deleteObject(tc.obj, "", settings)
tc.getObject(tc.obj, "", true)
versions := tc.listVersions()
require.Len(t, versions.DeleteMarker, 1)
for _, ver := range versions.DeleteMarker {
if ver.IsLatest {
tc.deleteObject(tc.obj, ver.ObjectInfo.VersionID(), settings)
}
}
resInfo, buffer := tc.getObject(tc.obj, "", false)
require.Equal(t, objV3Content, buffer)
require.Equal(t, objV3Info.VersionID(), resInfo.VersionID())
}
func TestNoVersioningDeleteObject(t *testing.T) {
tc := prepareContext(t)
tc.putObject([]byte("content obj1 v1"))
tc.putObject([]byte("content obj1 v2"))
settings, err := tc.layer.GetBucketSettings(tc.ctx, tc.bktInfo)
require.NoError(t, err)
tc.deleteObject(tc.obj, "", settings)
tc.getObject(tc.obj, "", true)
tc.checkListObjects()
}
func TestFilterVersionsByMarker(t *testing.T) {
n := 10
testOIDs := make([]oid.ID, n)
for i := 0; i < n; i++ {
testOIDs[i] = oidtest.ID()
}
for _, tc := range []struct {
name string
objects []*data.ExtendedObjectInfo
params *ListObjectVersionsParams
expected []*data.ExtendedObjectInfo
error bool
}{
{
name: "missed key marker",
objects: []*data.ExtendedObjectInfo{
{ObjectInfo: &data.ObjectInfo{Name: "obj0", ID: testOIDs[0]}},
{ObjectInfo: &data.ObjectInfo{Name: "obj0", ID: testOIDs[1]}},
},
params: &ListObjectVersionsParams{KeyMarker: "", VersionIDMarker: "dummy"},
expected: []*data.ExtendedObjectInfo{
{ObjectInfo: &data.ObjectInfo{Name: "obj0", ID: testOIDs[0]}},
{ObjectInfo: &data.ObjectInfo{Name: "obj0", ID: testOIDs[1]}},
},
},
{
name: "last version id",
objects: []*data.ExtendedObjectInfo{
{ObjectInfo: &data.ObjectInfo{Name: "obj0", ID: testOIDs[0]}},
{ObjectInfo: &data.ObjectInfo{Name: "obj0", ID: testOIDs[1]}},
},
params: &ListObjectVersionsParams{KeyMarker: "obj0", VersionIDMarker: testOIDs[1].EncodeToString()},
expected: []*data.ExtendedObjectInfo{},
},
{
name: "same name, different versions",
objects: []*data.ExtendedObjectInfo{
{ObjectInfo: &data.ObjectInfo{Name: "obj0", ID: testOIDs[0]}},
{ObjectInfo: &data.ObjectInfo{Name: "obj0", ID: testOIDs[1]}},
},
params: &ListObjectVersionsParams{KeyMarker: "obj0", VersionIDMarker: testOIDs[0].EncodeToString()},
expected: []*data.ExtendedObjectInfo{
{ObjectInfo: &data.ObjectInfo{Name: "obj0", ID: testOIDs[1]}},
},
},
{
name: "different name, different versions",
objects: []*data.ExtendedObjectInfo{
{ObjectInfo: &data.ObjectInfo{Name: "obj0", ID: testOIDs[0]}},
{ObjectInfo: &data.ObjectInfo{Name: "obj1", ID: testOIDs[1]}},
},
params: &ListObjectVersionsParams{KeyMarker: "obj0", VersionIDMarker: testOIDs[0].EncodeToString()},
expected: []*data.ExtendedObjectInfo{
{ObjectInfo: &data.ObjectInfo{Name: "obj1", ID: testOIDs[1]}},
},
},
{
name: "not matched name alphabetically less",
objects: []*data.ExtendedObjectInfo{
{ObjectInfo: &data.ObjectInfo{Name: "obj0", ID: testOIDs[0]}},
{ObjectInfo: &data.ObjectInfo{Name: "obj1", ID: testOIDs[1]}},
},
params: &ListObjectVersionsParams{KeyMarker: "obj", VersionIDMarker: ""},
expected: []*data.ExtendedObjectInfo{
{ObjectInfo: &data.ObjectInfo{Name: "obj0", ID: testOIDs[0]}},
{ObjectInfo: &data.ObjectInfo{Name: "obj1", ID: testOIDs[1]}},
},
},
{
name: "not matched name alphabetically less with dummy version id",
objects: []*data.ExtendedObjectInfo{
{ObjectInfo: &data.ObjectInfo{Name: "obj0", ID: testOIDs[0]}},
},
params: &ListObjectVersionsParams{KeyMarker: "obj", VersionIDMarker: "dummy"},
error: true,
},
{
name: "not matched name alphabetically greater",
objects: []*data.ExtendedObjectInfo{
{ObjectInfo: &data.ObjectInfo{Name: "obj0", ID: testOIDs[0]}},
{ObjectInfo: &data.ObjectInfo{Name: "obj1", ID: testOIDs[1]}},
},
params: &ListObjectVersionsParams{KeyMarker: "obj2", VersionIDMarker: testOIDs[2].EncodeToString()},
expected: []*data.ExtendedObjectInfo{},
},
{
name: "not found version id",
objects: []*data.ExtendedObjectInfo{
{ObjectInfo: &data.ObjectInfo{Name: "obj0", ID: testOIDs[0]}},
{ObjectInfo: &data.ObjectInfo{Name: "obj0", ID: testOIDs[1]}},
{ObjectInfo: &data.ObjectInfo{Name: "obj1", ID: testOIDs[2]}},
},
params: &ListObjectVersionsParams{KeyMarker: "obj0", VersionIDMarker: "dummy"},
error: true,
},
{
name: "not found version id, obj last",
objects: []*data.ExtendedObjectInfo{
{ObjectInfo: &data.ObjectInfo{Name: "obj0", ID: testOIDs[0]}},
{ObjectInfo: &data.ObjectInfo{Name: "obj0", ID: testOIDs[1]}},
},
params: &ListObjectVersionsParams{KeyMarker: "obj0", VersionIDMarker: "dummy"},
error: true,
},
{
name: "not found version id, obj last",
objects: []*data.ExtendedObjectInfo{
{ObjectInfo: &data.ObjectInfo{Name: "obj0", ID: testOIDs[0]}},
{ObjectInfo: &data.ObjectInfo{Name: "obj0", ID: testOIDs[1]}},
{ObjectInfo: &data.ObjectInfo{Name: "obj1", ID: testOIDs[2]}},
},
params: &ListObjectVersionsParams{KeyMarker: "obj0", VersionIDMarker: ""},
expected: []*data.ExtendedObjectInfo{
{ObjectInfo: &data.ObjectInfo{Name: "obj1", ID: testOIDs[2]}},
},
},
} {
t.Run(tc.name, func(t *testing.T) {
actual, err := filterVersionsByMarker(tc.objects, tc.params)
if tc.error {
require.Error(t, err)
} else {
require.NoError(t, err)
require.Equal(t, tc.expected, actual)
}
})
}
}