frostfs-s3-gw/api/layer/versioning_test.go

package layer

import (
	"bytes"
	"context"
	"io"
	"testing"

	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/frostfs"
	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
	bearertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer/test"
	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
	oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
	oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
	"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
	"github.com/stretchr/testify/require"
	"go.uber.org/zap"
)

func (tc *testContext) putObject(content []byte) *data.ObjectInfo {
	extObjInfo, err := tc.layer.PutObject(tc.ctx, &PutObjectParams{
		BktInfo: tc.bktInfo,
		Object:  tc.obj,
		Size:    ptr(uint64(len(content))),
		Reader:  bytes.NewReader(content),
		Header:  make(map[string]string),
	})
	require.NoError(tc.t, err)

	return extObjInfo.ObjectInfo
}

func (tc *testContext) getObject(objectName, versionID string, needError bool) (*data.ObjectInfo, []byte) {
	headPrm := &HeadObjectParams{
		BktInfo:   tc.bktInfo,
		Object:    objectName,
		VersionID: versionID,
	}
	objInfo, err := tc.layer.GetObjectInfo(tc.ctx, headPrm)
	if needError {
		require.Error(tc.t, err)
		return nil, nil
	}
	require.NoError(tc.t, err)

	objPayload, err := tc.layer.GetObject(tc.ctx, &GetObjectParams{
		ObjectInfo: objInfo,
		Versioned:  headPrm.Versioned(),
		BucketInfo: tc.bktInfo,
	})
	require.NoError(tc.t, err)

	payload, err := io.ReadAll(objPayload)
	require.NoError(tc.t, err)

	return objInfo, payload
}

func (tc *testContext) deleteObject(objectName, versionID string, settings *data.BucketSettings) {
	p := &DeleteObjectParams{
		BktInfo:  tc.bktInfo,
		Settings: settings,
		Objects: []*VersionedObject{
			{Name: objectName, VersionID: versionID},
		},
	}
	deletedObjects := tc.layer.DeleteObjects(tc.ctx, p)
	for _, obj := range deletedObjects {
		require.NoError(tc.t, obj.Error)
	}
}

func (tc *testContext) listObjectsV1() []*data.ExtendedNodeVersion {
	res, err := tc.layer.ListObjectsV1(tc.ctx, &ListObjectsParamsV1{
		ListObjectsParamsCommon: ListObjectsParamsCommon{
			BktInfo: tc.bktInfo,
			MaxKeys: 1000,
		},
	})
	require.NoError(tc.t, err)
	return res.Objects
}

func (tc *testContext) listObjectsV2() []*data.ExtendedNodeVersion {
	res, err := tc.layer.ListObjectsV2(tc.ctx, &ListObjectsParamsV2{
		ListObjectsParamsCommon: ListObjectsParamsCommon{
			BktInfo: tc.bktInfo,
			MaxKeys: 1000,
		},
	})
	require.NoError(tc.t, err)
	return res.Objects
}

func (tc *testContext) listVersions() *ListObjectVersionsInfo {
	res, err := tc.layer.ListObjectVersions(tc.ctx, &ListObjectVersionsParams{
		BktInfo: tc.bktInfo,
		MaxKeys: 1000,
	})
	require.NoError(tc.t, err)
	return res
}

func (tc *testContext) checkListObjects(ids ...oid.ID) {
	objs := tc.listObjectsV1()
	require.Equal(tc.t, len(ids), len(objs))
	for _, id := range ids {
		require.Contains(tc.t, ids, id)
	}

	objs = tc.listObjectsV2()
	require.Equal(tc.t, len(ids), len(objs))
	for _, id := range ids {
		require.Contains(tc.t, ids, id)
	}
}

func (tc *testContext) getObjectByID(objID oid.ID) *object.Object {
	for _, obj := range tc.testFrostFS.Objects() {
		id, _ := obj.ID()
		if id.Equals(objID) {
			return obj
		}
	}
	return nil
}

type testContext struct {
	t           *testing.T
	ctx         context.Context
	layer       *Layer
	bktInfo     *data.BucketInfo
	obj         string
	testFrostFS *TestFrostFS
}

func prepareContext(t *testing.T, cachesConfig ...*CachesConfig) *testContext {
	logger := zap.NewExample()

	key, err := keys.NewPrivateKey()
	require.NoError(t, err)

	bearerToken := bearertest.Token()
	require.NoError(t, bearerToken.Sign(key.PrivateKey))

	ctx := middleware.SetBox(context.Background(), &middleware.Box{AccessBox: &accessbox.Box{
		Gate: &accessbox.GateData{
			BearerToken: &bearerToken,
			GateKey:     key.PublicKey(),
		},
	}})
	tp := NewTestFrostFS(key)

	bktName := "testbucket1"
	res, err := tp.CreateContainer(ctx, frostfs.PrmContainerCreate{
		Name: bktName,
	})
	require.NoError(t, err)

	config := DefaultCachesConfigs(logger)
	if len(cachesConfig) != 0 {
		config = cachesConfig[0]
	}

	var owner user.ID
	user.IDFromKey(&owner, key.PrivateKey.PublicKey)

	layerCfg := &Config{
		Cache:       NewCache(config),
		AnonKey:     AnonymousKey{Key: key},
		TreeService: NewTreeService(),
		Features:    &FeatureSettingsMock{},
		GateOwner:   owner,
	}

	return &testContext{
		ctx:   ctx,
		layer: NewLayer(logger, tp, layerCfg),
		bktInfo: &data.BucketInfo{
			Name:                    bktName,
			Owner:                   owner,
			CID:                     res.ContainerID,
			HomomorphicHashDisabled: res.HomomorphicHashDisabled,
		},
		obj:         "obj1",
		t:           t,
		testFrostFS: tp,
	}
}

func TestSimpleVersioning(t *testing.T) {
	tc := prepareContext(t)
	err := tc.layer.PutBucketSettings(tc.ctx, &PutSettingsParams{
		BktInfo:  tc.bktInfo,
		Settings: &data.BucketSettings{Versioning: data.VersioningEnabled},
	})
	require.NoError(t, err)

	obj1Content1 := []byte("content obj1 v1")
	obj1v1 := tc.putObject(obj1Content1)

	obj1Content2 := []byte("content obj1 v2")
	obj1v2 := tc.putObject(obj1Content2)

	_, buffer2 := tc.getObject(tc.obj, "", false)
	require.Equal(t, obj1Content2, buffer2)

	_, buffer1 := tc.getObject(tc.obj, obj1v1.ID.EncodeToString(), false)
	require.Equal(t, obj1Content1, buffer1)

	tc.checkListObjects(obj1v2.ID)
}

func TestSimpleNoVersioning(t *testing.T) {
	tc := prepareContext(t)

	obj1Content1 := []byte("content obj1 v1")
	obj1v1 := tc.putObject(obj1Content1)

	obj1Content2 := []byte("content obj1 v2")
	obj1v2 := tc.putObject(obj1Content2)

	_, buffer2 := tc.getObject(tc.obj, "", false)
	require.Equal(t, obj1Content2, buffer2)

	tc.getObject(tc.obj, obj1v1.ID.EncodeToString(), true)
	tc.checkListObjects(obj1v2.ID)
}

func TestVersioningDeleteObject(t *testing.T) {
	tc := prepareContext(t)
	settings := &data.BucketSettings{Versioning: data.VersioningEnabled}
	err := tc.layer.PutBucketSettings(tc.ctx, &PutSettingsParams{
		BktInfo:  tc.bktInfo,
		Settings: settings,
	})
	require.NoError(t, err)

	tc.putObject([]byte("content obj1 v1"))
	tc.putObject([]byte("content obj1 v2"))

	tc.deleteObject(tc.obj, "", settings)
	tc.getObject(tc.obj, "", true)

	tc.checkListObjects()
}

func TestGetUnversioned(t *testing.T) {
	tc := prepareContext(t)

	objContent := []byte("content obj1 v1")
	objInfo := tc.putObject(objContent)

	settings := &data.BucketSettings{Versioning: data.VersioningUnversioned}
	err := tc.layer.PutBucketSettings(tc.ctx, &PutSettingsParams{
		BktInfo:  tc.bktInfo,
		Settings: settings,
	})
	require.NoError(t, err)

	resInfo, buffer := tc.getObject(tc.obj, data.UnversionedObjectVersionID, false)
	require.Equal(t, objContent, buffer)
	require.Equal(t, objInfo.VersionID(), resInfo.VersionID())
}

func TestVersioningDeleteSpecificObjectVersion(t *testing.T) {
	tc := prepareContext(t)
	settings := &data.BucketSettings{Versioning: data.VersioningEnabled}
	err := tc.layer.PutBucketSettings(tc.ctx, &PutSettingsParams{
		BktInfo:  tc.bktInfo,
		Settings: settings,
	})
	require.NoError(t, err)

	tc.putObject([]byte("content obj1 v1"))
	objV2Info := tc.putObject([]byte("content obj1 v2"))
	objV3Content := []byte("content obj1 v3")
	objV3Info := tc.putObject(objV3Content)

	tc.deleteObject(tc.obj, objV2Info.VersionID(), settings)
	tc.getObject(tc.obj, objV2Info.VersionID(), true)

	_, buffer3 := tc.getObject(tc.obj, "", false)
	require.Equal(t, objV3Content, buffer3)

	tc.deleteObject(tc.obj, "", settings)
	tc.getObject(tc.obj, "", true)

	versions := tc.listVersions()
	require.Len(t, versions.DeleteMarker, 1)
	for _, ver := range versions.DeleteMarker {
		if ver.IsLatest {
			tc.deleteObject(tc.obj, ver.NodeVersion.OID.EncodeToString(), settings)
		}
	}

	resInfo, buffer := tc.getObject(tc.obj, "", false)
	require.Equal(t, objV3Content, buffer)
	require.Equal(t, objV3Info.VersionID(), resInfo.VersionID())
}

func TestNoVersioningDeleteObject(t *testing.T) {
	tc := prepareContext(t)

	tc.putObject([]byte("content obj1 v1"))
	tc.putObject([]byte("content obj1 v2"))

	settings, err := tc.layer.GetBucketSettings(tc.ctx, tc.bktInfo)
	require.NoError(t, err)

	tc.deleteObject(tc.obj, "", settings)
	tc.getObject(tc.obj, "", true)
	tc.checkListObjects()
}

func TestFilterVersionsByMarker(t *testing.T) {
	n := 10
	testOIDs := make([]oid.ID, n)
	for i := 0; i < n; i++ {
		testOIDs[i] = oidtest.ID()
	}

	for _, tc := range []struct {
		name     string
		objects  []*data.ExtendedNodeVersion
		params   *ListObjectVersionsParams
		expected []*data.ExtendedNodeVersion
		error    bool
	}{
		{
			name: "missed key marker",
			objects: []*data.ExtendedNodeVersion{
				{NodeVersion: &data.NodeVersion{BaseNodeVersion: data.BaseNodeVersion{FilePath: "obj0", OID: testOIDs[0]}}},
				{NodeVersion: &data.NodeVersion{BaseNodeVersion: data.BaseNodeVersion{FilePath: "obj0", OID: testOIDs[1]}}},
			},
			params: &ListObjectVersionsParams{KeyMarker: "", VersionIDMarker: "dummy"},
			expected: []*data.ExtendedNodeVersion{
				{NodeVersion: &data.NodeVersion{BaseNodeVersion: data.BaseNodeVersion{FilePath: "obj0", OID: testOIDs[0]}}},
				{NodeVersion: &data.NodeVersion{BaseNodeVersion: data.BaseNodeVersion{FilePath: "obj0", OID: testOIDs[1]}}},
			},
		},
		{
			name: "last version id",
			objects: []*data.ExtendedNodeVersion{
				{NodeVersion: &data.NodeVersion{BaseNodeVersion: data.BaseNodeVersion{FilePath: "obj0", OID: testOIDs[0]}}},
				{NodeVersion: &data.NodeVersion{BaseNodeVersion: data.BaseNodeVersion{FilePath: "obj0", OID: testOIDs[1]}}},
			},
			params:   &ListObjectVersionsParams{KeyMarker: "obj0", VersionIDMarker: testOIDs[1].EncodeToString()},
			expected: []*data.ExtendedNodeVersion{},
		},
		{
			name: "same name, different versions",
			objects: []*data.ExtendedNodeVersion{
				{NodeVersion: &data.NodeVersion{BaseNodeVersion: data.BaseNodeVersion{FilePath: "obj0", OID: testOIDs[0]}}},
				{NodeVersion: &data.NodeVersion{BaseNodeVersion: data.BaseNodeVersion{FilePath: "obj0", OID: testOIDs[1]}}},
			},
			params: &ListObjectVersionsParams{KeyMarker: "obj0", VersionIDMarker: testOIDs[0].EncodeToString()},
			expected: []*data.ExtendedNodeVersion{
				{NodeVersion: &data.NodeVersion{BaseNodeVersion: data.BaseNodeVersion{FilePath: "obj0", OID: testOIDs[1]}}},
			},
		},
		{
			name: "different name, different versions",
			objects: []*data.ExtendedNodeVersion{
				{NodeVersion: &data.NodeVersion{BaseNodeVersion: data.BaseNodeVersion{FilePath: "obj0", OID: testOIDs[0]}}},
				{NodeVersion: &data.NodeVersion{BaseNodeVersion: data.BaseNodeVersion{FilePath: "obj1", OID: testOIDs[1]}}},
			},
			params: &ListObjectVersionsParams{KeyMarker: "obj0", VersionIDMarker: testOIDs[0].EncodeToString()},
			expected: []*data.ExtendedNodeVersion{
				{NodeVersion: &data.NodeVersion{BaseNodeVersion: data.BaseNodeVersion{FilePath: "obj1", OID: testOIDs[1]}}},
			},
		},
		{
			name: "not matched name alphabetically less",
			objects: []*data.ExtendedNodeVersion{
				{NodeVersion: &data.NodeVersion{BaseNodeVersion: data.BaseNodeVersion{FilePath: "obj0", OID: testOIDs[0]}}},
				{NodeVersion: &data.NodeVersion{BaseNodeVersion: data.BaseNodeVersion{FilePath: "obj1", OID: testOIDs[1]}}},
			},
			params: &ListObjectVersionsParams{KeyMarker: "obj", VersionIDMarker: ""},
			expected: []*data.ExtendedNodeVersion{
				{NodeVersion: &data.NodeVersion{BaseNodeVersion: data.BaseNodeVersion{FilePath: "obj0", OID: testOIDs[0]}}},
				{NodeVersion: &data.NodeVersion{BaseNodeVersion: data.BaseNodeVersion{FilePath: "obj1", OID: testOIDs[1]}}},
			},
		},
		{
			name: "not matched name alphabetically less with dummy version id",
			objects: []*data.ExtendedNodeVersion{
				{NodeVersion: &data.NodeVersion{BaseNodeVersion: data.BaseNodeVersion{FilePath: "obj0", OID: testOIDs[0]}}},
			},
			params: &ListObjectVersionsParams{KeyMarker: "obj", VersionIDMarker: "dummy"},
			error:  true,
		},
		{
			name: "not matched name alphabetically greater",
			objects: []*data.ExtendedNodeVersion{
				{NodeVersion: &data.NodeVersion{BaseNodeVersion: data.BaseNodeVersion{FilePath: "obj0", OID: testOIDs[0]}}},
				{NodeVersion: &data.NodeVersion{BaseNodeVersion: data.BaseNodeVersion{FilePath: "obj1", OID: testOIDs[1]}}},
			},
			params:   &ListObjectVersionsParams{KeyMarker: "obj2", VersionIDMarker: testOIDs[2].EncodeToString()},
			expected: []*data.ExtendedNodeVersion{},
		},
		{
			name: "not found version id",
			objects: []*data.ExtendedNodeVersion{
				{NodeVersion: &data.NodeVersion{BaseNodeVersion: data.BaseNodeVersion{FilePath: "obj0", OID: testOIDs[0]}}},
				{NodeVersion: &data.NodeVersion{BaseNodeVersion: data.BaseNodeVersion{FilePath: "obj0", OID: testOIDs[1]}}},
				{NodeVersion: &data.NodeVersion{BaseNodeVersion: data.BaseNodeVersion{FilePath: "obj1", OID: testOIDs[2]}}},
			},
			params: &ListObjectVersionsParams{KeyMarker: "obj0", VersionIDMarker: "dummy"},
			error:  true,
		},
		{
			name: "not found version id, obj last",
			objects: []*data.ExtendedNodeVersion{
				{NodeVersion: &data.NodeVersion{BaseNodeVersion: data.BaseNodeVersion{FilePath: "obj0", OID: testOIDs[0]}}},
				{NodeVersion: &data.NodeVersion{BaseNodeVersion: data.BaseNodeVersion{FilePath: "obj0", OID: testOIDs[1]}}},
			},
			params: &ListObjectVersionsParams{KeyMarker: "obj0", VersionIDMarker: "dummy"},
			error:  true,
		},
		{
			name: "not found version id, obj last",
			objects: []*data.ExtendedNodeVersion{
				{NodeVersion: &data.NodeVersion{BaseNodeVersion: data.BaseNodeVersion{FilePath: "obj0", OID: testOIDs[0]}}},
				{NodeVersion: &data.NodeVersion{BaseNodeVersion: data.BaseNodeVersion{FilePath: "obj0", OID: testOIDs[1]}}},
				{NodeVersion: &data.NodeVersion{BaseNodeVersion: data.BaseNodeVersion{FilePath: "obj1", OID: testOIDs[2]}}},
			},
			params: &ListObjectVersionsParams{KeyMarker: "obj0", VersionIDMarker: ""},
			expected: []*data.ExtendedNodeVersion{
				{NodeVersion: &data.NodeVersion{BaseNodeVersion: data.BaseNodeVersion{FilePath: "obj1", OID: testOIDs[2]}}},
			},
		},
	} {
		t.Run(tc.name, func(t *testing.T) {
			actual, err := filterVersionsByMarker(tc.objects, tc.params)
			if tc.error {
				require.Error(t, err)
			} else {
				require.NoError(t, err)
				require.Equal(t, tc.expected, actual)
			}
		})
	}
}