[#165] Fix real object size in listing

Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
This commit is contained in:
Denis Kirillov 2024-01-22 14:11:12 +03:00
parent 88f1acbdfc
commit fafe4af529
5 changed files with 111 additions and 10 deletions

View file

@ -65,7 +65,7 @@ type BaseNodeVersion struct {
ParenID uint64 ParenID uint64
OID oid.ID OID oid.ID
Timestamp uint64 Timestamp uint64
Size uint64 // todo discuss if it is possible don't support correct obj size for ola objects Size uint64
ETag string ETag string
MD5 string MD5 string
FilePath string FilePath string
@ -86,9 +86,10 @@ func (v BaseNodeVersion) IsFilledExtra() bool {
return v.Created != nil && v.Owner != nil return v.Created != nil && v.Owner != nil
} }
func (v *BaseNodeVersion) FillExtra(objInfo *ObjectInfo) { func (v *BaseNodeVersion) FillExtra(owner *user.ID, created *time.Time, realSize uint64) {
v.Owner = &objInfo.Owner v.Owner = owner
v.Created = &objInfo.Created v.Created = created
v.Size = realSize
} }
type ObjectTaggingInfo struct { type ObjectTaggingInfo struct {

View file

@ -191,7 +191,7 @@ func encodeToObjectAttributesResponse(info *data.ObjectInfo, p *GetObjectAttribu
case storageClass: case storageClass:
resp.StorageClass = api.DefaultStorageClass resp.StorageClass = api.DefaultStorageClass
case objectSize: case objectSize:
resp.ObjectSize = info.Size // todo probably we need to use GetObjectSize resp.ObjectSize = info.Size
case checksum: case checksum:
checksumBytes, err := hex.DecodeString(info.HashSum) checksumBytes, err := hex.DecodeString(info.HashSum)
if err != nil { if err != nil {

View file

@ -200,10 +200,6 @@ func fillContents(src []*data.ExtendedNodeVersion, encode string, fetchOwner, md
StorageClass: api.DefaultStorageClass, StorageClass: api.DefaultStorageClass,
} }
//if size, err := layer.GetObjectSize(obj); err == nil {
// res.Size = size
//}
if fetchOwner { if fetchOwner {
owner := obj.NodeVersion.Owner.String() owner := obj.NodeVersion.Owner.String()
res.Owner = &Owner{ res.Owner = &Owner{

View file

@ -7,6 +7,7 @@ import (
"net/url" "net/url"
"sort" "sort"
"strconv" "strconv"
"strings"
"testing" "testing"
"time" "time"
@ -15,6 +16,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/pkg/service/tree"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"go.uber.org/zap/zaptest" "go.uber.org/zap/zaptest"
) )
@ -65,6 +67,102 @@ func TestListObjectNullVersions(t *testing.T) {
require.Equal(t, data.UnversionedObjectVersionID, result.Version[1].VersionID) require.Equal(t, data.UnversionedObjectVersionID, result.Version[1].VersionID)
} }
func TestListObjectsWithOldTreeNodes(t *testing.T) {
hc := prepareHandlerContext(t)
bktName, objName := "bucket-versioning-enabled", "object"
bktInfo := createTestBucket(hc, bktName)
srcEnc, err := encryption.NewParams([]byte("1234567890qwertyuiopasdfghjklzxc"))
require.NoError(t, err)
objInfo := createTestObject(hc, bktInfo, objName, *srcEnc)
_ = objInfo
prm := &tree.GetNodesParams{
BktInfo: bktInfo,
TreeID: "version",
Path: []string{objName},
LatestOnly: true,
AllAttrs: true,
}
nodes, err := hc.treeMock.GetNodes(hc.Context(), prm)
require.NoError(t, err)
require.Len(t, nodes, 1)
node := nodes[0]
meta := make(map[string]string, len(node.GetMeta()))
for _, m := range node.GetMeta() {
if m.GetKey() != "Created" && m.GetKey() != "Owner" {
meta[m.GetKey()] = string(m.GetValue())
}
}
err = hc.treeMock.MoveNode(hc.Context(), bktInfo, "version", node.GetNodeID(), node.GetParentID(), meta)
require.NoError(t, err)
list := listObjectsV1(hc, bktName, "", "", "", -1)
require.Len(t, list.Contents, 1)
realSize, err := layer.GetObjectSize(objInfo)
require.NoError(t, err)
require.Equal(t, objInfo.Owner.EncodeToString(), list.Contents[0].Owner.ID)
require.Equal(t, objInfo.Created.UTC().Format(time.RFC3339), list.Contents[0].LastModified)
require.Equal(t, realSize, list.Contents[0].Size)
}
func prepareObjects(hc *handlerContext, bktInfo *data.BucketInfo, prefix string, size int) []string {
treeID := "version"
parentID := uint64(0)
if prefix != "" {
for _, filename := range strings.Split(prefix, "/") {
nodeID, err := hc.treeMock.AddNode(hc.Context(), bktInfo, treeID, parentID, map[string]string{
"FileName": filename,
})
require.NoError(hc.t, err)
parentID = nodeID
}
prefix += "/"
}
objects := make([]string, size)
for i := range objects {
filename := "object" + strconv.Itoa(i)
filepath := prefix + filename
prm := layer.PrmObjectCreate{
Container: bktInfo.CID,
Filepath: filepath,
Payload: nil,
}
id, err := hc.tp.CreateObject(hc.Context(), prm)
require.NoError(hc.t, err)
newVersion := &data.NodeVersion{
BaseNodeVersion: data.BaseNodeVersion{
OID: id,
ETag: "12345678",
FilePath: filepath,
},
IsUnversioned: true,
IsCombined: false,
}
_, err = hc.treeMock.AddNodeBase(hc.Context(), bktInfo, treeID, parentID, map[string]string{
"OID": newVersion.OID.EncodeToString(),
"FileName": filename,
"IsUnversioned": "true",
}, false)
require.NoError(hc.t, err)
objects[i] = filepath
}
hc.treeMock.Sort()
sort.Strings(objects)
return objects
}
func TestListObjectsContextCanceled(t *testing.T) { func TestListObjectsContextCanceled(t *testing.T) {
layerCfg := layer.DefaultCachesConfigs(zaptest.NewLogger(t)) layerCfg := layer.DefaultCachesConfigs(zaptest.NewLogger(t))
layerCfg.SessionList.Lifetime = time.Hour layerCfg.SessionList.Lifetime = time.Hour

View file

@ -535,7 +535,13 @@ func (n *layer) initWorkerPool(ctx context.Context, size int, p commonVersionsLi
} }
} }
node.NodeVersion.FillExtra(oi) realSize, err := GetObjectSize(oi)
if err != nil {
reqLog.Debug("failed to get real object size", zap.Error(err))
realSize = oi.Size
}
node.NodeVersion.FillExtra(&oi.Owner, &oi.Created, realSize)
select { select {
case <-ctx.Done(): case <-ctx.Done():