Denis Kirillov
5ee73fad6a
All checks were successful
/ DCO (pull_request) Successful in 4m40s
/ Vulncheck (pull_request) Successful in 5m43s
/ Lint (pull_request) Successful in 10m27s
/ Tests (1.20) (pull_request) Successful in 5m41s
/ Tests (1.21) (pull_request) Successful in 5m47s
/ Builds (1.20) (pull_request) Successful in 1m9s
/ Builds (1.21) (pull_request) Successful in 4m7s
Despite the spec https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectVersions.html#API_ListObjectVersions_ResponseElements says that "When the number of responses exceeds the value of MaxKeys, NextVersionIdMarker specifies the first object version not returned that satisfies the search criteria. Use this value for the version-id-marker request parameter in a subsequent request." the actual behavior of AWS S3 is returning NextVersionIdMarker as the last returned object version Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
444 lines
13 KiB
Go
444 lines
13 KiB
Go
package layer
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"io"
|
|
"testing"
|
|
|
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
|
bearertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer/test"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
|
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
|
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
|
"github.com/stretchr/testify/require"
|
|
"go.uber.org/zap"
|
|
)
|
|
|
|
func (tc *testContext) putObject(content []byte) *data.ObjectInfo {
|
|
extObjInfo, err := tc.layer.PutObject(tc.ctx, &PutObjectParams{
|
|
BktInfo: tc.bktInfo,
|
|
Object: tc.obj,
|
|
Size: uint64(len(content)),
|
|
Reader: bytes.NewReader(content),
|
|
Header: make(map[string]string),
|
|
})
|
|
require.NoError(tc.t, err)
|
|
|
|
return extObjInfo.ObjectInfo
|
|
}
|
|
|
|
func (tc *testContext) getObject(objectName, versionID string, needError bool) (*data.ObjectInfo, []byte) {
|
|
headPrm := &HeadObjectParams{
|
|
BktInfo: tc.bktInfo,
|
|
Object: objectName,
|
|
VersionID: versionID,
|
|
}
|
|
objInfo, err := tc.layer.GetObjectInfo(tc.ctx, headPrm)
|
|
if needError {
|
|
require.Error(tc.t, err)
|
|
return nil, nil
|
|
}
|
|
require.NoError(tc.t, err)
|
|
|
|
objPayload, err := tc.layer.GetObject(tc.ctx, &GetObjectParams{
|
|
ObjectInfo: objInfo,
|
|
Versioned: headPrm.Versioned(),
|
|
BucketInfo: tc.bktInfo,
|
|
})
|
|
require.NoError(tc.t, err)
|
|
|
|
payload, err := io.ReadAll(objPayload)
|
|
require.NoError(tc.t, err)
|
|
|
|
return objInfo, payload
|
|
}
|
|
|
|
func (tc *testContext) deleteObject(objectName, versionID string, settings *data.BucketSettings) {
|
|
p := &DeleteObjectParams{
|
|
BktInfo: tc.bktInfo,
|
|
Settings: settings,
|
|
Objects: []*VersionedObject{
|
|
{Name: objectName, VersionID: versionID},
|
|
},
|
|
}
|
|
deletedObjects := tc.layer.DeleteObjects(tc.ctx, p)
|
|
for _, obj := range deletedObjects {
|
|
require.NoError(tc.t, obj.Error)
|
|
}
|
|
}
|
|
|
|
func (tc *testContext) listObjectsV1() []*data.ObjectInfo {
|
|
res, err := tc.layer.ListObjectsV1(tc.ctx, &ListObjectsParamsV1{
|
|
ListObjectsParamsCommon: ListObjectsParamsCommon{
|
|
BktInfo: tc.bktInfo,
|
|
MaxKeys: 1000,
|
|
},
|
|
})
|
|
require.NoError(tc.t, err)
|
|
return res.Objects
|
|
}
|
|
|
|
func (tc *testContext) listObjectsV2() []*data.ObjectInfo {
|
|
res, err := tc.layer.ListObjectsV2(tc.ctx, &ListObjectsParamsV2{
|
|
ListObjectsParamsCommon: ListObjectsParamsCommon{
|
|
BktInfo: tc.bktInfo,
|
|
MaxKeys: 1000,
|
|
},
|
|
})
|
|
require.NoError(tc.t, err)
|
|
return res.Objects
|
|
}
|
|
|
|
func (tc *testContext) listVersions() *ListObjectVersionsInfo {
|
|
res, err := tc.layer.ListObjectVersions(tc.ctx, &ListObjectVersionsParams{
|
|
BktInfo: tc.bktInfo,
|
|
MaxKeys: 1000,
|
|
})
|
|
require.NoError(tc.t, err)
|
|
return res
|
|
}
|
|
|
|
func (tc *testContext) checkListObjects(ids ...oid.ID) {
|
|
objs := tc.listObjectsV1()
|
|
require.Equal(tc.t, len(ids), len(objs))
|
|
for _, id := range ids {
|
|
require.Contains(tc.t, ids, id)
|
|
}
|
|
|
|
objs = tc.listObjectsV2()
|
|
require.Equal(tc.t, len(ids), len(objs))
|
|
for _, id := range ids {
|
|
require.Contains(tc.t, ids, id)
|
|
}
|
|
}
|
|
|
|
func (tc *testContext) getObjectByID(objID oid.ID) *object.Object {
|
|
for _, obj := range tc.testFrostFS.Objects() {
|
|
id, _ := obj.ID()
|
|
if id.Equals(objID) {
|
|
return obj
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
type testContext struct {
|
|
t *testing.T
|
|
ctx context.Context
|
|
layer Client
|
|
bktInfo *data.BucketInfo
|
|
obj string
|
|
testFrostFS *TestFrostFS
|
|
}
|
|
|
|
func prepareContext(t *testing.T, cachesConfig ...*CachesConfig) *testContext {
|
|
logger := zap.NewExample()
|
|
|
|
key, err := keys.NewPrivateKey()
|
|
require.NoError(t, err)
|
|
|
|
bearerToken := bearertest.Token()
|
|
require.NoError(t, bearerToken.Sign(key.PrivateKey))
|
|
|
|
ctx := middleware.SetBoxData(context.Background(), &accessbox.Box{
|
|
Gate: &accessbox.GateData{
|
|
BearerToken: &bearerToken,
|
|
GateKey: key.PublicKey(),
|
|
},
|
|
})
|
|
tp := NewTestFrostFS(key)
|
|
|
|
bktName := "testbucket1"
|
|
res, err := tp.CreateContainer(ctx, PrmContainerCreate{
|
|
Name: bktName,
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
config := DefaultCachesConfigs(logger)
|
|
if len(cachesConfig) != 0 {
|
|
config = cachesConfig[0]
|
|
}
|
|
|
|
var owner user.ID
|
|
user.IDFromKey(&owner, key.PrivateKey.PublicKey)
|
|
|
|
layerCfg := &Config{
|
|
Caches: config,
|
|
AnonKey: AnonymousKey{Key: key},
|
|
TreeService: NewTreeService(),
|
|
Features: &FeatureSettingsMock{},
|
|
}
|
|
|
|
return &testContext{
|
|
ctx: ctx,
|
|
layer: NewLayer(logger, tp, layerCfg),
|
|
bktInfo: &data.BucketInfo{
|
|
Name: bktName,
|
|
Owner: owner,
|
|
CID: res.ContainerID,
|
|
HomomorphicHashDisabled: res.HomomorphicHashDisabled,
|
|
},
|
|
obj: "obj1",
|
|
t: t,
|
|
testFrostFS: tp,
|
|
}
|
|
}
|
|
|
|
func TestSimpleVersioning(t *testing.T) {
|
|
tc := prepareContext(t)
|
|
err := tc.layer.PutBucketSettings(tc.ctx, &PutSettingsParams{
|
|
BktInfo: tc.bktInfo,
|
|
Settings: &data.BucketSettings{Versioning: data.VersioningEnabled},
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
obj1Content1 := []byte("content obj1 v1")
|
|
obj1v1 := tc.putObject(obj1Content1)
|
|
|
|
obj1Content2 := []byte("content obj1 v2")
|
|
obj1v2 := tc.putObject(obj1Content2)
|
|
|
|
_, buffer2 := tc.getObject(tc.obj, "", false)
|
|
require.Equal(t, obj1Content2, buffer2)
|
|
|
|
_, buffer1 := tc.getObject(tc.obj, obj1v1.ID.EncodeToString(), false)
|
|
require.Equal(t, obj1Content1, buffer1)
|
|
|
|
tc.checkListObjects(obj1v2.ID)
|
|
}
|
|
|
|
func TestSimpleNoVersioning(t *testing.T) {
|
|
tc := prepareContext(t)
|
|
|
|
obj1Content1 := []byte("content obj1 v1")
|
|
obj1v1 := tc.putObject(obj1Content1)
|
|
|
|
obj1Content2 := []byte("content obj1 v2")
|
|
obj1v2 := tc.putObject(obj1Content2)
|
|
|
|
_, buffer2 := tc.getObject(tc.obj, "", false)
|
|
require.Equal(t, obj1Content2, buffer2)
|
|
|
|
tc.getObject(tc.obj, obj1v1.ID.EncodeToString(), true)
|
|
tc.checkListObjects(obj1v2.ID)
|
|
}
|
|
|
|
func TestVersioningDeleteObject(t *testing.T) {
|
|
tc := prepareContext(t)
|
|
settings := &data.BucketSettings{Versioning: data.VersioningEnabled}
|
|
err := tc.layer.PutBucketSettings(tc.ctx, &PutSettingsParams{
|
|
BktInfo: tc.bktInfo,
|
|
Settings: settings,
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
tc.putObject([]byte("content obj1 v1"))
|
|
tc.putObject([]byte("content obj1 v2"))
|
|
|
|
tc.deleteObject(tc.obj, "", settings)
|
|
tc.getObject(tc.obj, "", true)
|
|
|
|
tc.checkListObjects()
|
|
}
|
|
|
|
func TestGetUnversioned(t *testing.T) {
|
|
tc := prepareContext(t)
|
|
|
|
objContent := []byte("content obj1 v1")
|
|
objInfo := tc.putObject(objContent)
|
|
|
|
settings := &data.BucketSettings{Versioning: data.VersioningUnversioned}
|
|
err := tc.layer.PutBucketSettings(tc.ctx, &PutSettingsParams{
|
|
BktInfo: tc.bktInfo,
|
|
Settings: settings,
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
resInfo, buffer := tc.getObject(tc.obj, data.UnversionedObjectVersionID, false)
|
|
require.Equal(t, objContent, buffer)
|
|
require.Equal(t, objInfo.VersionID(), resInfo.VersionID())
|
|
}
|
|
|
|
func TestVersioningDeleteSpecificObjectVersion(t *testing.T) {
|
|
tc := prepareContext(t)
|
|
settings := &data.BucketSettings{Versioning: data.VersioningEnabled}
|
|
err := tc.layer.PutBucketSettings(tc.ctx, &PutSettingsParams{
|
|
BktInfo: tc.bktInfo,
|
|
Settings: settings,
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
tc.putObject([]byte("content obj1 v1"))
|
|
objV2Info := tc.putObject([]byte("content obj1 v2"))
|
|
objV3Content := []byte("content obj1 v3")
|
|
objV3Info := tc.putObject(objV3Content)
|
|
|
|
tc.deleteObject(tc.obj, objV2Info.VersionID(), settings)
|
|
tc.getObject(tc.obj, objV2Info.VersionID(), true)
|
|
|
|
_, buffer3 := tc.getObject(tc.obj, "", false)
|
|
require.Equal(t, objV3Content, buffer3)
|
|
|
|
tc.deleteObject(tc.obj, "", settings)
|
|
tc.getObject(tc.obj, "", true)
|
|
|
|
versions := tc.listVersions()
|
|
for _, ver := range versions.DeleteMarker {
|
|
if ver.IsLatest {
|
|
tc.deleteObject(tc.obj, ver.ObjectInfo.VersionID(), settings)
|
|
}
|
|
}
|
|
|
|
resInfo, buffer := tc.getObject(tc.obj, "", false)
|
|
require.Equal(t, objV3Content, buffer)
|
|
require.Equal(t, objV3Info.VersionID(), resInfo.VersionID())
|
|
}
|
|
|
|
func TestNoVersioningDeleteObject(t *testing.T) {
|
|
tc := prepareContext(t)
|
|
|
|
tc.putObject([]byte("content obj1 v1"))
|
|
tc.putObject([]byte("content obj1 v2"))
|
|
|
|
settings, err := tc.layer.GetBucketSettings(tc.ctx, tc.bktInfo)
|
|
require.NoError(t, err)
|
|
|
|
tc.deleteObject(tc.obj, "", settings)
|
|
tc.getObject(tc.obj, "", true)
|
|
tc.checkListObjects()
|
|
}
|
|
|
|
func TestFilterVersionsByMarker(t *testing.T) {
|
|
n := 10
|
|
testOIDs := make([]oid.ID, n)
|
|
for i := 0; i < n; i++ {
|
|
testOIDs[i] = oidtest.ID()
|
|
}
|
|
|
|
for _, tc := range []struct {
|
|
name string
|
|
objects []*data.ExtendedObjectInfo
|
|
params *ListObjectVersionsParams
|
|
expected []*data.ExtendedObjectInfo
|
|
error bool
|
|
}{
|
|
{
|
|
name: "missed key marker",
|
|
objects: []*data.ExtendedObjectInfo{
|
|
{ObjectInfo: &data.ObjectInfo{Name: "obj0", ID: testOIDs[0]}},
|
|
{ObjectInfo: &data.ObjectInfo{Name: "obj0", ID: testOIDs[1]}},
|
|
},
|
|
params: &ListObjectVersionsParams{KeyMarker: "", VersionIDMarker: "dummy"},
|
|
expected: []*data.ExtendedObjectInfo{
|
|
{ObjectInfo: &data.ObjectInfo{Name: "obj0", ID: testOIDs[0]}},
|
|
{ObjectInfo: &data.ObjectInfo{Name: "obj0", ID: testOIDs[1]}},
|
|
},
|
|
},
|
|
{
|
|
name: "last version id",
|
|
objects: []*data.ExtendedObjectInfo{
|
|
{ObjectInfo: &data.ObjectInfo{Name: "obj0", ID: testOIDs[0]}},
|
|
{ObjectInfo: &data.ObjectInfo{Name: "obj0", ID: testOIDs[1]}},
|
|
},
|
|
params: &ListObjectVersionsParams{KeyMarker: "obj0", VersionIDMarker: testOIDs[1].EncodeToString()},
|
|
expected: []*data.ExtendedObjectInfo{},
|
|
},
|
|
{
|
|
name: "same name, different versions",
|
|
objects: []*data.ExtendedObjectInfo{
|
|
{ObjectInfo: &data.ObjectInfo{Name: "obj0", ID: testOIDs[0]}},
|
|
{ObjectInfo: &data.ObjectInfo{Name: "obj0", ID: testOIDs[1]}},
|
|
},
|
|
params: &ListObjectVersionsParams{KeyMarker: "obj0", VersionIDMarker: testOIDs[0].EncodeToString()},
|
|
expected: []*data.ExtendedObjectInfo{
|
|
{ObjectInfo: &data.ObjectInfo{Name: "obj0", ID: testOIDs[1]}},
|
|
},
|
|
},
|
|
{
|
|
name: "different name, different versions",
|
|
objects: []*data.ExtendedObjectInfo{
|
|
{ObjectInfo: &data.ObjectInfo{Name: "obj0", ID: testOIDs[0]}},
|
|
{ObjectInfo: &data.ObjectInfo{Name: "obj1", ID: testOIDs[1]}},
|
|
},
|
|
params: &ListObjectVersionsParams{KeyMarker: "obj0", VersionIDMarker: testOIDs[0].EncodeToString()},
|
|
expected: []*data.ExtendedObjectInfo{
|
|
{ObjectInfo: &data.ObjectInfo{Name: "obj1", ID: testOIDs[1]}},
|
|
},
|
|
},
|
|
{
|
|
name: "not matched name alphabetically less",
|
|
objects: []*data.ExtendedObjectInfo{
|
|
{ObjectInfo: &data.ObjectInfo{Name: "obj0", ID: testOIDs[0]}},
|
|
{ObjectInfo: &data.ObjectInfo{Name: "obj1", ID: testOIDs[1]}},
|
|
},
|
|
params: &ListObjectVersionsParams{KeyMarker: "obj", VersionIDMarker: ""},
|
|
expected: []*data.ExtendedObjectInfo{
|
|
{ObjectInfo: &data.ObjectInfo{Name: "obj0", ID: testOIDs[0]}},
|
|
{ObjectInfo: &data.ObjectInfo{Name: "obj1", ID: testOIDs[1]}},
|
|
},
|
|
},
|
|
{
|
|
name: "not matched name alphabetically less with dummy version id",
|
|
objects: []*data.ExtendedObjectInfo{
|
|
{ObjectInfo: &data.ObjectInfo{Name: "obj0", ID: testOIDs[0]}},
|
|
},
|
|
params: &ListObjectVersionsParams{KeyMarker: "obj", VersionIDMarker: "dummy"},
|
|
error: true,
|
|
},
|
|
{
|
|
name: "not matched name alphabetically greater",
|
|
objects: []*data.ExtendedObjectInfo{
|
|
{ObjectInfo: &data.ObjectInfo{Name: "obj0", ID: testOIDs[0]}},
|
|
{ObjectInfo: &data.ObjectInfo{Name: "obj1", ID: testOIDs[1]}},
|
|
},
|
|
params: &ListObjectVersionsParams{KeyMarker: "obj2", VersionIDMarker: testOIDs[2].EncodeToString()},
|
|
expected: []*data.ExtendedObjectInfo{},
|
|
},
|
|
{
|
|
name: "not found version id",
|
|
objects: []*data.ExtendedObjectInfo{
|
|
{ObjectInfo: &data.ObjectInfo{Name: "obj0", ID: testOIDs[0]}},
|
|
{ObjectInfo: &data.ObjectInfo{Name: "obj0", ID: testOIDs[1]}},
|
|
{ObjectInfo: &data.ObjectInfo{Name: "obj1", ID: testOIDs[2]}},
|
|
},
|
|
params: &ListObjectVersionsParams{KeyMarker: "obj0", VersionIDMarker: "dummy"},
|
|
error: true,
|
|
},
|
|
{
|
|
name: "not found version id, obj last",
|
|
objects: []*data.ExtendedObjectInfo{
|
|
{ObjectInfo: &data.ObjectInfo{Name: "obj0", ID: testOIDs[0]}},
|
|
{ObjectInfo: &data.ObjectInfo{Name: "obj0", ID: testOIDs[1]}},
|
|
},
|
|
params: &ListObjectVersionsParams{KeyMarker: "obj0", VersionIDMarker: "dummy"},
|
|
error: true,
|
|
},
|
|
{
|
|
name: "not found version id, obj last",
|
|
objects: []*data.ExtendedObjectInfo{
|
|
{ObjectInfo: &data.ObjectInfo{Name: "obj0", ID: testOIDs[0]}},
|
|
{ObjectInfo: &data.ObjectInfo{Name: "obj0", ID: testOIDs[1]}},
|
|
{ObjectInfo: &data.ObjectInfo{Name: "obj1", ID: testOIDs[2]}},
|
|
},
|
|
params: &ListObjectVersionsParams{KeyMarker: "obj0", VersionIDMarker: ""},
|
|
expected: []*data.ExtendedObjectInfo{
|
|
{ObjectInfo: &data.ObjectInfo{Name: "obj1", ID: testOIDs[2]}},
|
|
},
|
|
},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
actual, err := filterVersionsByMarker(tc.objects, tc.params)
|
|
if tc.error {
|
|
require.Error(t, err)
|
|
} else {
|
|
require.NoError(t, err)
|
|
require.Equal(t, tc.expected, actual)
|
|
}
|
|
})
|
|
}
|
|
}
|