package layer import ( "bytes" "context" "io" "testing" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox" bearertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer/test" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/stretchr/testify/require" "go.uber.org/zap" ) func (tc *testContext) putObject(content []byte) *data.ObjectInfo { extObjInfo, err := tc.layer.PutObject(tc.ctx, &PutObjectParams{ BktInfo: tc.bktInfo, Object: tc.obj, Size: uint64(len(content)), Reader: bytes.NewReader(content), Header: make(map[string]string), }) require.NoError(tc.t, err) return extObjInfo.ObjectInfo } func (tc *testContext) getObject(objectName, versionID string, needError bool) (*data.ObjectInfo, []byte) { headPrm := &HeadObjectParams{ BktInfo: tc.bktInfo, Object: objectName, VersionID: versionID, } objInfo, err := tc.layer.GetObjectInfo(tc.ctx, headPrm) if needError { require.Error(tc.t, err) return nil, nil } require.NoError(tc.t, err) objPayload, err := tc.layer.GetObject(tc.ctx, &GetObjectParams{ ObjectInfo: objInfo, Versioned: headPrm.Versioned(), BucketInfo: tc.bktInfo, }) require.NoError(tc.t, err) payload, err := io.ReadAll(objPayload) require.NoError(tc.t, err) return objInfo, payload } func (tc *testContext) deleteObject(objectName, versionID string, settings *data.BucketSettings) { p := &DeleteObjectParams{ BktInfo: tc.bktInfo, Settings: settings, Objects: []*VersionedObject{ {Name: objectName, VersionID: versionID}, }, } deletedObjects := tc.layer.DeleteObjects(tc.ctx, p) for _, obj := range deletedObjects { require.NoError(tc.t, obj.Error) } } func (tc *testContext) listObjectsV1() []*data.ExtendedNodeVersion { res, err := tc.layer.ListObjectsV1(tc.ctx, &ListObjectsParamsV1{ ListObjectsParamsCommon: ListObjectsParamsCommon{ BktInfo: tc.bktInfo, MaxKeys: 1000, }, }) require.NoError(tc.t, err) return res.Objects } func (tc *testContext) listObjectsV2() []*data.ExtendedNodeVersion { res, err := tc.layer.ListObjectsV2(tc.ctx, &ListObjectsParamsV2{ ListObjectsParamsCommon: ListObjectsParamsCommon{ BktInfo: tc.bktInfo, MaxKeys: 1000, }, }) require.NoError(tc.t, err) return res.Objects } func (tc *testContext) listVersions() *ListObjectVersionsInfo { res, err := tc.layer.ListObjectVersions(tc.ctx, &ListObjectVersionsParams{ BktInfo: tc.bktInfo, MaxKeys: 1000, }) require.NoError(tc.t, err) return res } func (tc *testContext) checkListObjects(ids ...oid.ID) { objs := tc.listObjectsV1() require.Equal(tc.t, len(ids), len(objs)) for _, id := range ids { require.Contains(tc.t, ids, id) } objs = tc.listObjectsV2() require.Equal(tc.t, len(ids), len(objs)) for _, id := range ids { require.Contains(tc.t, ids, id) } } func (tc *testContext) getObjectByID(objID oid.ID) *object.Object { for _, obj := range tc.testFrostFS.Objects() { id, _ := obj.ID() if id.Equals(objID) { return obj } } return nil } type testContext struct { t *testing.T ctx context.Context layer Client bktInfo *data.BucketInfo obj string testFrostFS *TestFrostFS } func prepareContext(t *testing.T, cachesConfig ...*CachesConfig) *testContext { logger := zap.NewExample() key, err := keys.NewPrivateKey() require.NoError(t, err) bearerToken := bearertest.Token() require.NoError(t, bearerToken.Sign(key.PrivateKey)) ctx := middleware.SetBoxData(context.Background(), &accessbox.Box{ Gate: &accessbox.GateData{ BearerToken: &bearerToken, GateKey: key.PublicKey(), }, }) tp := NewTestFrostFS(key) bktName := "testbucket1" res, err := tp.CreateContainer(ctx, PrmContainerCreate{ Name: bktName, }) require.NoError(t, err) config := DefaultCachesConfigs(logger) if len(cachesConfig) != 0 { config = cachesConfig[0] } var owner user.ID user.IDFromKey(&owner, key.PrivateKey.PublicKey) layerCfg := &Config{ Caches: config, AnonKey: AnonymousKey{Key: key}, TreeService: NewTreeService(), Features: &FeatureSettingsMock{}, GateOwner: owner, } return &testContext{ ctx: ctx, layer: NewLayer(logger, tp, layerCfg), bktInfo: &data.BucketInfo{ Name: bktName, Owner: owner, CID: res.ContainerID, HomomorphicHashDisabled: res.HomomorphicHashDisabled, }, obj: "obj1", t: t, testFrostFS: tp, } } func TestSimpleVersioning(t *testing.T) { tc := prepareContext(t) err := tc.layer.PutBucketSettings(tc.ctx, &PutSettingsParams{ BktInfo: tc.bktInfo, Settings: &data.BucketSettings{Versioning: data.VersioningEnabled}, }) require.NoError(t, err) obj1Content1 := []byte("content obj1 v1") obj1v1 := tc.putObject(obj1Content1) obj1Content2 := []byte("content obj1 v2") obj1v2 := tc.putObject(obj1Content2) _, buffer2 := tc.getObject(tc.obj, "", false) require.Equal(t, obj1Content2, buffer2) _, buffer1 := tc.getObject(tc.obj, obj1v1.ID.EncodeToString(), false) require.Equal(t, obj1Content1, buffer1) tc.checkListObjects(obj1v2.ID) } func TestSimpleNoVersioning(t *testing.T) { tc := prepareContext(t) obj1Content1 := []byte("content obj1 v1") obj1v1 := tc.putObject(obj1Content1) obj1Content2 := []byte("content obj1 v2") obj1v2 := tc.putObject(obj1Content2) _, buffer2 := tc.getObject(tc.obj, "", false) require.Equal(t, obj1Content2, buffer2) tc.getObject(tc.obj, obj1v1.ID.EncodeToString(), true) tc.checkListObjects(obj1v2.ID) } func TestVersioningDeleteObject(t *testing.T) { tc := prepareContext(t) settings := &data.BucketSettings{Versioning: data.VersioningEnabled} err := tc.layer.PutBucketSettings(tc.ctx, &PutSettingsParams{ BktInfo: tc.bktInfo, Settings: settings, }) require.NoError(t, err) tc.putObject([]byte("content obj1 v1")) tc.putObject([]byte("content obj1 v2")) tc.deleteObject(tc.obj, "", settings) tc.getObject(tc.obj, "", true) tc.checkListObjects() } func TestGetUnversioned(t *testing.T) { tc := prepareContext(t) objContent := []byte("content obj1 v1") objInfo := tc.putObject(objContent) settings := &data.BucketSettings{Versioning: data.VersioningUnversioned} err := tc.layer.PutBucketSettings(tc.ctx, &PutSettingsParams{ BktInfo: tc.bktInfo, Settings: settings, }) require.NoError(t, err) resInfo, buffer := tc.getObject(tc.obj, data.UnversionedObjectVersionID, false) require.Equal(t, objContent, buffer) require.Equal(t, objInfo.VersionID(), resInfo.VersionID()) } func TestVersioningDeleteSpecificObjectVersion(t *testing.T) { tc := prepareContext(t) settings := &data.BucketSettings{Versioning: data.VersioningEnabled} err := tc.layer.PutBucketSettings(tc.ctx, &PutSettingsParams{ BktInfo: tc.bktInfo, Settings: settings, }) require.NoError(t, err) tc.putObject([]byte("content obj1 v1")) objV2Info := tc.putObject([]byte("content obj1 v2")) objV3Content := []byte("content obj1 v3") objV3Info := tc.putObject(objV3Content) tc.deleteObject(tc.obj, objV2Info.VersionID(), settings) tc.getObject(tc.obj, objV2Info.VersionID(), true) _, buffer3 := tc.getObject(tc.obj, "", false) require.Equal(t, objV3Content, buffer3) tc.deleteObject(tc.obj, "", settings) tc.getObject(tc.obj, "", true) versions := tc.listVersions() require.Len(t, versions.DeleteMarker, 1) for _, ver := range versions.DeleteMarker { if ver.IsLatest { tc.deleteObject(tc.obj, ver.NodeVersion.OID.EncodeToString(), settings) } } resInfo, buffer := tc.getObject(tc.obj, "", false) require.Equal(t, objV3Content, buffer) require.Equal(t, objV3Info.VersionID(), resInfo.VersionID()) } func TestNoVersioningDeleteObject(t *testing.T) { tc := prepareContext(t) tc.putObject([]byte("content obj1 v1")) tc.putObject([]byte("content obj1 v2")) settings, err := tc.layer.GetBucketSettings(tc.ctx, tc.bktInfo) require.NoError(t, err) tc.deleteObject(tc.obj, "", settings) tc.getObject(tc.obj, "", true) tc.checkListObjects() } func TestFilterVersionsByMarker(t *testing.T) { n := 10 testOIDs := make([]oid.ID, n) for i := 0; i < n; i++ { testOIDs[i] = oidtest.ID() } for _, tc := range []struct { name string objects []*data.ExtendedNodeVersion params *ListObjectVersionsParams expected []*data.ExtendedNodeVersion error bool }{ { name: "missed key marker", objects: []*data.ExtendedNodeVersion{ {NodeVersion: &data.NodeVersion{BaseNodeVersion: data.BaseNodeVersion{FilePath: "obj0", OID: testOIDs[0]}}}, {NodeVersion: &data.NodeVersion{BaseNodeVersion: data.BaseNodeVersion{FilePath: "obj0", OID: testOIDs[1]}}}, }, params: &ListObjectVersionsParams{KeyMarker: "", VersionIDMarker: "dummy"}, expected: []*data.ExtendedNodeVersion{ {NodeVersion: &data.NodeVersion{BaseNodeVersion: data.BaseNodeVersion{FilePath: "obj0", OID: testOIDs[0]}}}, {NodeVersion: &data.NodeVersion{BaseNodeVersion: data.BaseNodeVersion{FilePath: "obj0", OID: testOIDs[1]}}}, }, }, { name: "last version id", objects: []*data.ExtendedNodeVersion{ {NodeVersion: &data.NodeVersion{BaseNodeVersion: data.BaseNodeVersion{FilePath: "obj0", OID: testOIDs[0]}}}, {NodeVersion: &data.NodeVersion{BaseNodeVersion: data.BaseNodeVersion{FilePath: "obj0", OID: testOIDs[1]}}}, }, params: &ListObjectVersionsParams{KeyMarker: "obj0", VersionIDMarker: testOIDs[1].EncodeToString()}, expected: []*data.ExtendedNodeVersion{}, }, { name: "same name, different versions", objects: []*data.ExtendedNodeVersion{ {NodeVersion: &data.NodeVersion{BaseNodeVersion: data.BaseNodeVersion{FilePath: "obj0", OID: testOIDs[0]}}}, {NodeVersion: &data.NodeVersion{BaseNodeVersion: data.BaseNodeVersion{FilePath: "obj0", OID: testOIDs[1]}}}, }, params: &ListObjectVersionsParams{KeyMarker: "obj0", VersionIDMarker: testOIDs[0].EncodeToString()}, expected: []*data.ExtendedNodeVersion{ {NodeVersion: &data.NodeVersion{BaseNodeVersion: data.BaseNodeVersion{FilePath: "obj0", OID: testOIDs[1]}}}, }, }, { name: "different name, different versions", objects: []*data.ExtendedNodeVersion{ {NodeVersion: &data.NodeVersion{BaseNodeVersion: data.BaseNodeVersion{FilePath: "obj0", OID: testOIDs[0]}}}, {NodeVersion: &data.NodeVersion{BaseNodeVersion: data.BaseNodeVersion{FilePath: "obj1", OID: testOIDs[1]}}}, }, params: &ListObjectVersionsParams{KeyMarker: "obj0", VersionIDMarker: testOIDs[0].EncodeToString()}, expected: []*data.ExtendedNodeVersion{ {NodeVersion: &data.NodeVersion{BaseNodeVersion: data.BaseNodeVersion{FilePath: "obj1", OID: testOIDs[1]}}}, }, }, { name: "not matched name alphabetically less", objects: []*data.ExtendedNodeVersion{ {NodeVersion: &data.NodeVersion{BaseNodeVersion: data.BaseNodeVersion{FilePath: "obj0", OID: testOIDs[0]}}}, {NodeVersion: &data.NodeVersion{BaseNodeVersion: data.BaseNodeVersion{FilePath: "obj1", OID: testOIDs[1]}}}, }, params: &ListObjectVersionsParams{KeyMarker: "obj", VersionIDMarker: ""}, expected: []*data.ExtendedNodeVersion{ {NodeVersion: &data.NodeVersion{BaseNodeVersion: data.BaseNodeVersion{FilePath: "obj0", OID: testOIDs[0]}}}, {NodeVersion: &data.NodeVersion{BaseNodeVersion: data.BaseNodeVersion{FilePath: "obj1", OID: testOIDs[1]}}}, }, }, { name: "not matched name alphabetically less with dummy version id", objects: []*data.ExtendedNodeVersion{ {NodeVersion: &data.NodeVersion{BaseNodeVersion: data.BaseNodeVersion{FilePath: "obj0", OID: testOIDs[0]}}}, }, params: &ListObjectVersionsParams{KeyMarker: "obj", VersionIDMarker: "dummy"}, error: true, }, { name: "not matched name alphabetically greater", objects: []*data.ExtendedNodeVersion{ {NodeVersion: &data.NodeVersion{BaseNodeVersion: data.BaseNodeVersion{FilePath: "obj0", OID: testOIDs[0]}}}, {NodeVersion: &data.NodeVersion{BaseNodeVersion: data.BaseNodeVersion{FilePath: "obj1", OID: testOIDs[1]}}}, }, params: &ListObjectVersionsParams{KeyMarker: "obj2", VersionIDMarker: testOIDs[2].EncodeToString()}, expected: []*data.ExtendedNodeVersion{}, }, { name: "not found version id", objects: []*data.ExtendedNodeVersion{ {NodeVersion: &data.NodeVersion{BaseNodeVersion: data.BaseNodeVersion{FilePath: "obj0", OID: testOIDs[0]}}}, {NodeVersion: &data.NodeVersion{BaseNodeVersion: data.BaseNodeVersion{FilePath: "obj0", OID: testOIDs[1]}}}, {NodeVersion: &data.NodeVersion{BaseNodeVersion: data.BaseNodeVersion{FilePath: "obj1", OID: testOIDs[2]}}}, }, params: &ListObjectVersionsParams{KeyMarker: "obj0", VersionIDMarker: "dummy"}, error: true, }, { name: "not found version id, obj last", objects: []*data.ExtendedNodeVersion{ {NodeVersion: &data.NodeVersion{BaseNodeVersion: data.BaseNodeVersion{FilePath: "obj0", OID: testOIDs[0]}}}, {NodeVersion: &data.NodeVersion{BaseNodeVersion: data.BaseNodeVersion{FilePath: "obj0", OID: testOIDs[1]}}}, }, params: &ListObjectVersionsParams{KeyMarker: "obj0", VersionIDMarker: "dummy"}, error: true, }, { name: "not found version id, obj last", objects: []*data.ExtendedNodeVersion{ {NodeVersion: &data.NodeVersion{BaseNodeVersion: data.BaseNodeVersion{FilePath: "obj0", OID: testOIDs[0]}}}, {NodeVersion: &data.NodeVersion{BaseNodeVersion: data.BaseNodeVersion{FilePath: "obj0", OID: testOIDs[1]}}}, {NodeVersion: &data.NodeVersion{BaseNodeVersion: data.BaseNodeVersion{FilePath: "obj1", OID: testOIDs[2]}}}, }, params: &ListObjectVersionsParams{KeyMarker: "obj0", VersionIDMarker: ""}, expected: []*data.ExtendedNodeVersion{ {NodeVersion: &data.NodeVersion{BaseNodeVersion: data.BaseNodeVersion{FilePath: "obj1", OID: testOIDs[2]}}}, }, }, } { t.Run(tc.name, func(t *testing.T) { actual, err := filterVersionsByMarker(tc.objects, tc.params) if tc.error { require.Error(t, err) } else { require.NoError(t, err) require.Equal(t, tc.expected, actual) } }) } }