forked from TrueCloudLab/frostfs-s3-gw
569 lines
13 KiB
Go
569 lines
13 KiB
Go
package tree
|
|
|
|
import (
|
|
"context"
|
|
"io"
|
|
"sort"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
|
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
|
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
|
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
|
|
usertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user/test"
|
|
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
|
"github.com/stretchr/testify/require"
|
|
"go.uber.org/zap"
|
|
"go.uber.org/zap/zaptest"
|
|
)
|
|
|
|
func TestLockConfigurationEncoding(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
name string
|
|
encoded string
|
|
expectedEncoded string
|
|
expected data.ObjectLockConfiguration
|
|
error bool
|
|
}{
|
|
{
|
|
name: "empty",
|
|
encoded: "",
|
|
expectedEncoded: "",
|
|
expected: data.ObjectLockConfiguration{},
|
|
},
|
|
{
|
|
name: "Enabled",
|
|
encoded: "Enabled",
|
|
expectedEncoded: "Enabled",
|
|
expected: data.ObjectLockConfiguration{
|
|
ObjectLockEnabled: "Enabled",
|
|
},
|
|
},
|
|
{
|
|
name: "Fully enabled",
|
|
encoded: "Enabled,10,COMPLIANCE,",
|
|
expectedEncoded: "Enabled,10,COMPLIANCE,0",
|
|
expected: data.ObjectLockConfiguration{
|
|
ObjectLockEnabled: "Enabled",
|
|
Rule: &data.ObjectLockRule{
|
|
DefaultRetention: &data.DefaultRetention{
|
|
Days: 10,
|
|
Mode: "COMPLIANCE",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
name: "Missing numbers",
|
|
encoded: "Enabled,,COMPLIANCE,",
|
|
expectedEncoded: "Enabled,0,COMPLIANCE,0",
|
|
expected: data.ObjectLockConfiguration{
|
|
ObjectLockEnabled: "Enabled",
|
|
Rule: &data.ObjectLockRule{
|
|
DefaultRetention: &data.DefaultRetention{
|
|
Mode: "COMPLIANCE",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
name: "Missing all",
|
|
encoded: ",,,",
|
|
expectedEncoded: ",0,,0",
|
|
expected: data.ObjectLockConfiguration{Rule: &data.ObjectLockRule{DefaultRetention: &data.DefaultRetention{}}},
|
|
},
|
|
{
|
|
name: "Invalid args",
|
|
encoded: ",,",
|
|
error: true,
|
|
},
|
|
{
|
|
name: "Invalid days",
|
|
encoded: ",a,,",
|
|
error: true,
|
|
},
|
|
{
|
|
name: "Invalid years",
|
|
encoded: ",,,b",
|
|
error: true,
|
|
},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
lockConfiguration, err := parseLockConfiguration(tc.encoded)
|
|
if tc.error {
|
|
require.Error(t, err)
|
|
return
|
|
}
|
|
|
|
require.NoError(t, err)
|
|
require.Equal(t, tc.expected, *lockConfiguration)
|
|
|
|
encoded := encodeLockConfiguration(lockConfiguration)
|
|
require.Equal(t, tc.expectedEncoded, encoded)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestTreeServiceSettings(t *testing.T) {
|
|
ctx := context.Background()
|
|
|
|
memCli, err := NewTreeServiceClientMemory()
|
|
require.NoError(t, err)
|
|
treeService := NewTree(memCli, zaptest.NewLogger(t))
|
|
|
|
bktInfo := &data.BucketInfo{
|
|
CID: cidtest.ID(),
|
|
}
|
|
|
|
key, err := keys.NewPrivateKey()
|
|
require.NoError(t, err)
|
|
|
|
settings := &data.BucketSettings{
|
|
Versioning: data.Versioning{
|
|
VersioningStatus: data.VersioningEnabled,
|
|
MFADeleteStatus: data.MFADeleteDisabled,
|
|
},
|
|
LockConfiguration: &data.ObjectLockConfiguration{
|
|
ObjectLockEnabled: "Enabled",
|
|
Rule: &data.ObjectLockRule{
|
|
DefaultRetention: &data.DefaultRetention{
|
|
Days: 1,
|
|
Mode: "mode",
|
|
},
|
|
},
|
|
},
|
|
OwnerKey: key.PublicKey(),
|
|
}
|
|
|
|
err = treeService.PutSettingsNode(ctx, bktInfo, settings)
|
|
require.NoError(t, err)
|
|
|
|
storedSettings, err := treeService.GetSettingsNode(ctx, bktInfo)
|
|
require.NoError(t, err)
|
|
require.Equal(t, settings, storedSettings)
|
|
}
|
|
|
|
func TestTreeServiceAddVersion(t *testing.T) {
|
|
ctx := context.Background()
|
|
|
|
memCli, err := NewTreeServiceClientMemory()
|
|
require.NoError(t, err)
|
|
treeService := NewTree(memCli, zaptest.NewLogger(t))
|
|
|
|
bktInfo := &data.BucketInfo{
|
|
CID: cidtest.ID(),
|
|
}
|
|
|
|
userID := usertest.ID()
|
|
|
|
now := time.Now()
|
|
version := &data.NodeVersion{
|
|
BaseNodeVersion: data.BaseNodeVersion{
|
|
OID: oidtest.ID(),
|
|
Size: 10,
|
|
ETag: "etag",
|
|
FilePath: "path/to/version",
|
|
Owner: &userID,
|
|
Created: &now,
|
|
},
|
|
IsUnversioned: true,
|
|
}
|
|
|
|
nodeID, _, err := treeService.AddVersion(ctx, bktInfo, version)
|
|
require.NoError(t, err)
|
|
|
|
storedNode, err := treeService.GetUnversioned(ctx, bktInfo, "path/to/version")
|
|
require.NoError(t, err)
|
|
require.Equal(t, nodeID, storedNode.ID)
|
|
require.Equal(t, version.BaseNodeVersion.Size, storedNode.Size)
|
|
require.Equal(t, version.BaseNodeVersion.ETag, storedNode.ETag)
|
|
require.Equal(t, version.BaseNodeVersion.ETag, storedNode.ETag)
|
|
require.Equal(t, version.BaseNodeVersion.FilePath, storedNode.FilePath)
|
|
require.Equal(t, version.BaseNodeVersion.OID, storedNode.OID)
|
|
|
|
versions, err := treeService.GetVersions(ctx, bktInfo, "path/to/version")
|
|
require.NoError(t, err)
|
|
require.Len(t, versions, 1)
|
|
require.Equal(t, storedNode, versions[0])
|
|
}
|
|
|
|
func TestGetLatestNode(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
name string
|
|
nodes []NodeResponse
|
|
expectedNodeID uint64
|
|
error bool
|
|
}{
|
|
{
|
|
name: "empty",
|
|
nodes: []NodeResponse{},
|
|
error: true,
|
|
},
|
|
{
|
|
name: "one node of the object version",
|
|
nodes: []NodeResponse{
|
|
nodeResponse{
|
|
nodeID: 1,
|
|
parentID: 0,
|
|
timestamp: 1,
|
|
meta: []nodeMeta{
|
|
{
|
|
key: oidKV,
|
|
value: []byte(oidtest.ID().String()),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
expectedNodeID: 1,
|
|
},
|
|
{
|
|
name: "one node of the object version and one node of the secondary object",
|
|
nodes: []NodeResponse{
|
|
nodeResponse{
|
|
nodeID: 2,
|
|
parentID: 0,
|
|
timestamp: 3,
|
|
meta: []nodeMeta{},
|
|
},
|
|
nodeResponse{
|
|
nodeID: 1,
|
|
parentID: 0,
|
|
timestamp: 1,
|
|
meta: []nodeMeta{
|
|
{
|
|
key: oidKV,
|
|
value: []byte(oidtest.ID().String()),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
expectedNodeID: 1,
|
|
},
|
|
{
|
|
name: "all nodes represent a secondary object",
|
|
nodes: []NodeResponse{
|
|
nodeResponse{
|
|
nodeID: 2,
|
|
parentID: 0,
|
|
timestamp: 3,
|
|
meta: []nodeMeta{},
|
|
},
|
|
nodeResponse{
|
|
nodeID: 4,
|
|
parentID: 0,
|
|
timestamp: 5,
|
|
meta: []nodeMeta{},
|
|
},
|
|
},
|
|
error: true,
|
|
},
|
|
{
|
|
name: "several nodes of different types and with different timestamp",
|
|
nodes: []NodeResponse{
|
|
nodeResponse{
|
|
nodeID: 1,
|
|
parentID: 0,
|
|
timestamp: 1,
|
|
meta: []nodeMeta{
|
|
{
|
|
key: oidKV,
|
|
value: []byte(oidtest.ID().String()),
|
|
},
|
|
},
|
|
},
|
|
nodeResponse{
|
|
nodeID: 3,
|
|
parentID: 0,
|
|
timestamp: 3,
|
|
meta: []nodeMeta{},
|
|
},
|
|
nodeResponse{
|
|
nodeID: 4,
|
|
parentID: 0,
|
|
timestamp: 4,
|
|
meta: []nodeMeta{
|
|
{
|
|
key: oidKV,
|
|
value: []byte(oidtest.ID().String()),
|
|
},
|
|
},
|
|
},
|
|
nodeResponse{
|
|
nodeID: 6,
|
|
parentID: 0,
|
|
timestamp: 6,
|
|
meta: []nodeMeta{},
|
|
},
|
|
},
|
|
expectedNodeID: 4,
|
|
},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
actualNode, err := getLatestVersionNode(tc.nodes)
|
|
if tc.error {
|
|
require.Error(t, err)
|
|
return
|
|
}
|
|
|
|
require.NoError(t, err)
|
|
require.EqualValues(t, []uint64{tc.expectedNodeID}, actualNode.GetNodeID())
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestSplitTreeMultiparts(t *testing.T) {
|
|
ctx := context.Background()
|
|
|
|
memCli, err := NewTreeServiceClientMemory()
|
|
require.NoError(t, err)
|
|
treeService := NewTree(memCli, zaptest.NewLogger(t))
|
|
|
|
bktInfo := &data.BucketInfo{
|
|
CID: cidtest.ID(),
|
|
}
|
|
|
|
multipartInfo := &data.MultipartInfo{
|
|
Key: "multipart",
|
|
UploadID: "id",
|
|
Meta: map[string]string{},
|
|
Owner: usertest.ID(),
|
|
}
|
|
|
|
err = treeService.CreateMultipartUpload(ctx, bktInfo, multipartInfo)
|
|
require.NoError(t, err)
|
|
|
|
multipartInfo, err = treeService.GetMultipartUpload(ctx, bktInfo, multipartInfo.Key, multipartInfo.UploadID)
|
|
require.NoError(t, err)
|
|
|
|
var objIDs []oid.ID
|
|
for i := 0; i < 2; i++ {
|
|
objID := oidtest.ID()
|
|
_, err = memCli.AddNode(ctx, bktInfo, systemTree, multipartInfo.ID, map[string]string{
|
|
partNumberKV: "1",
|
|
oidKV: objID.EncodeToString(),
|
|
ownerKV: usertest.ID().EncodeToString(),
|
|
})
|
|
require.NoError(t, err)
|
|
objIDs = append(objIDs, objID)
|
|
}
|
|
|
|
parts, err := treeService.GetParts(ctx, bktInfo, multipartInfo.ID)
|
|
require.NoError(t, err)
|
|
require.Len(t, parts, 2)
|
|
|
|
objToDeletes, err := treeService.AddPart(ctx, bktInfo, multipartInfo.ID, &data.PartInfo{
|
|
Key: multipartInfo.Key,
|
|
UploadID: multipartInfo.UploadID,
|
|
Number: 1,
|
|
OID: oidtest.ID(),
|
|
})
|
|
require.NoError(t, err)
|
|
require.EqualValues(t, objIDs, objToDeletes, "oids to delete mismatched")
|
|
|
|
parts, err = treeService.GetParts(ctx, bktInfo, multipartInfo.ID)
|
|
require.NoError(t, err)
|
|
require.Len(t, parts, 1)
|
|
}
|
|
|
|
func TestVersionsByPrefixStreamImpl_Next(t *testing.T) {
|
|
ctx := context.Background()
|
|
|
|
memCli, err := NewTreeServiceClientMemory()
|
|
require.NoError(t, err)
|
|
treeService := NewTree(memCli, zaptest.NewLogger(t))
|
|
bktInfo := &data.BucketInfo{
|
|
CID: cidtest.ID(),
|
|
}
|
|
ownerID := usertest.ID()
|
|
created := time.Now()
|
|
versions := []*data.NodeVersion{
|
|
{
|
|
BaseNodeVersion: data.BaseNodeVersion{
|
|
ID: 1,
|
|
OID: oidtest.ID(),
|
|
FilePath: "foo",
|
|
Owner: &ownerID,
|
|
Created: &created,
|
|
},
|
|
},
|
|
{
|
|
BaseNodeVersion: data.BaseNodeVersion{
|
|
ID: 2,
|
|
OID: oidtest.ID(),
|
|
FilePath: "bar",
|
|
Owner: &ownerID,
|
|
Created: &created,
|
|
},
|
|
},
|
|
{
|
|
BaseNodeVersion: data.BaseNodeVersion{
|
|
ID: 3,
|
|
OID: oidtest.ID(),
|
|
FilePath: "test",
|
|
Owner: &ownerID,
|
|
Created: &created,
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, v := range versions {
|
|
_, _, err = treeService.AddVersion(ctx, bktInfo, v)
|
|
require.NoError(t, err)
|
|
}
|
|
|
|
sort.Slice(versions, func(i, j int) bool {
|
|
return versions[i].FilePath < versions[j].FilePath
|
|
})
|
|
|
|
t.Run("basic", func(t *testing.T) {
|
|
stream, err := treeService.InitVersionsByPrefixStream(ctx, bktInfo, "", false)
|
|
require.NoError(t, err)
|
|
|
|
for i := range len(versions) {
|
|
node, err := stream.Next(ctx)
|
|
require.NoError(t, err)
|
|
require.Equal(t, versions[i].ID, node.ID)
|
|
require.Equal(t, versions[i].FilePath, node.FilePath)
|
|
}
|
|
|
|
node, err := stream.Next(ctx)
|
|
require.Nil(t, node)
|
|
require.ErrorIs(t, err, io.EOF)
|
|
})
|
|
|
|
t.Run("context cancel", func(t *testing.T) {
|
|
stream, err := treeService.InitVersionsByPrefixStream(ctx, bktInfo, "", false)
|
|
require.NoError(t, err)
|
|
cancelCtx, cancel := context.WithCancel(ctx)
|
|
cancel()
|
|
|
|
node, err := stream.Next(cancelCtx)
|
|
require.Nil(t, node)
|
|
require.ErrorIs(t, err, context.Canceled)
|
|
})
|
|
}
|
|
|
|
func TestCheckTreeNode(t *testing.T) {
|
|
treeNodes := []*treeNode{
|
|
// foo/
|
|
{
|
|
ID: []uint64{1},
|
|
ParentID: []uint64{0},
|
|
TimeStamp: []uint64{1},
|
|
Meta: map[string]string{
|
|
"FileName": "foo",
|
|
},
|
|
},
|
|
// foo/ant
|
|
{
|
|
ID: []uint64{2},
|
|
ParentID: []uint64{1},
|
|
TimeStamp: []uint64{1},
|
|
Meta: map[string]string{
|
|
"FileName": "ant",
|
|
"UploadId": "d",
|
|
},
|
|
},
|
|
// foo/bar
|
|
{
|
|
ID: []uint64{3},
|
|
ParentID: []uint64{1},
|
|
TimeStamp: []uint64{1},
|
|
Meta: map[string]string{
|
|
"FileName": "bar",
|
|
"UploadId": "c",
|
|
},
|
|
},
|
|
// foo/finished
|
|
{
|
|
ID: []uint64{4},
|
|
ParentID: []uint64{1},
|
|
TimeStamp: []uint64{1},
|
|
Meta: map[string]string{
|
|
"FileName": "finished",
|
|
"UploadId": "e",
|
|
"Finished": "True",
|
|
},
|
|
},
|
|
// hello/
|
|
{
|
|
ID: []uint64{5},
|
|
ParentID: []uint64{0},
|
|
TimeStamp: []uint64{1},
|
|
Meta: map[string]string{
|
|
"FileName": "hello",
|
|
},
|
|
},
|
|
// hello/world
|
|
{
|
|
ID: []uint64{6},
|
|
ParentID: []uint64{5},
|
|
TimeStamp: []uint64{1},
|
|
Meta: map[string]string{
|
|
"FileName": "world",
|
|
"UploadId": "a",
|
|
},
|
|
},
|
|
// hello/world
|
|
{
|
|
ID: []uint64{7},
|
|
ParentID: []uint64{5},
|
|
TimeStamp: []uint64{1},
|
|
Meta: map[string]string{
|
|
"FileName": "world",
|
|
"UploadId": "b",
|
|
},
|
|
},
|
|
}
|
|
|
|
info := multipartInfoStream{
|
|
log: zap.NewNop(),
|
|
rootID: []uint64{0},
|
|
}
|
|
|
|
t.Run("without markers", func(t *testing.T) {
|
|
info.nodePaths = make(map[uint64]string)
|
|
results := make([]bool, 0, len(treeNodes))
|
|
for _, node := range treeNodes {
|
|
_, valid := info.checkTreeNode(node)
|
|
results = append(results, valid)
|
|
}
|
|
require.Equal(t, []bool{false, true, true, false, false, true, true}, results)
|
|
})
|
|
|
|
t.Run("with prefix", func(t *testing.T) {
|
|
info.nodePaths = make(map[uint64]string)
|
|
info.prefix = "hello"
|
|
info.headPrefix = ""
|
|
results := make([]bool, 0, len(treeNodes))
|
|
for _, node := range treeNodes {
|
|
_, valid := info.checkTreeNode(node)
|
|
results = append(results, valid)
|
|
}
|
|
require.Equal(t, []bool{false, false, false, false, false, true, true}, results)
|
|
})
|
|
|
|
t.Run("with key marker", func(t *testing.T) {
|
|
info.nodePaths = make(map[uint64]string)
|
|
info.keyMarker = "foo/bar"
|
|
results := make([]bool, 0, len(treeNodes))
|
|
for _, node := range treeNodes {
|
|
_, valid := info.checkTreeNode(node)
|
|
results = append(results, valid)
|
|
}
|
|
require.Equal(t, []bool{false, false, false, false, false, true, true}, results)
|
|
})
|
|
|
|
t.Run("with key and upload id markers", func(t *testing.T) {
|
|
info.nodePaths = make(map[uint64]string)
|
|
info.keyMarker = "hello/world"
|
|
info.uploadID = "a"
|
|
results := make([]bool, 0, len(treeNodes))
|
|
for _, node := range treeNodes {
|
|
_, valid := info.checkTreeNode(node)
|
|
results = append(results, valid)
|
|
}
|
|
require.Equal(t, []bool{false, false, false, false, false, false, true}, results)
|
|
})
|
|
}
|