diff --git a/api/data/tree.go b/api/data/tree.go index 277f32d..0164062 100644 --- a/api/data/tree.go +++ b/api/data/tree.go @@ -65,7 +65,7 @@ type BaseNodeVersion struct { ParenID uint64 OID oid.ID Timestamp uint64 - Size uint64 // todo discuss if it is possible don't support correct obj size for ola objects + Size uint64 ETag string MD5 string FilePath string @@ -86,9 +86,10 @@ func (v BaseNodeVersion) IsFilledExtra() bool { return v.Created != nil && v.Owner != nil } -func (v *BaseNodeVersion) FillExtra(objInfo *ObjectInfo) { - v.Owner = &objInfo.Owner - v.Created = &objInfo.Created +func (v *BaseNodeVersion) FillExtra(owner *user.ID, created *time.Time, realSize uint64) { + v.Owner = owner + v.Created = created + v.Size = realSize } type ObjectTaggingInfo struct { diff --git a/api/handler/attributes.go b/api/handler/attributes.go index 1a2c46d..a5e07b3 100644 --- a/api/handler/attributes.go +++ b/api/handler/attributes.go @@ -191,7 +191,7 @@ func encodeToObjectAttributesResponse(info *data.ObjectInfo, p *GetObjectAttribu case storageClass: resp.StorageClass = api.DefaultStorageClass case objectSize: - resp.ObjectSize = info.Size // todo probably we need to use GetObjectSize + resp.ObjectSize = info.Size case checksum: checksumBytes, err := hex.DecodeString(info.HashSum) if err != nil { diff --git a/api/handler/object_list.go b/api/handler/object_list.go index a386569..870b20e 100644 --- a/api/handler/object_list.go +++ b/api/handler/object_list.go @@ -200,10 +200,6 @@ func fillContents(src []*data.ExtendedNodeVersion, encode string, fetchOwner, md StorageClass: api.DefaultStorageClass, } - //if size, err := layer.GetObjectSize(obj); err == nil { - // res.Size = size - //} - if fetchOwner { owner := obj.NodeVersion.Owner.String() res.Owner = &Owner{ diff --git a/api/handler/object_list_test.go b/api/handler/object_list_test.go index c10e98a..e7e9795 100644 --- a/api/handler/object_list_test.go +++ b/api/handler/object_list_test.go @@ -7,6 +7,7 @@ import ( "net/url" "sort" "strconv" + "strings" "testing" "time" @@ -15,6 +16,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/pkg/service/tree" "github.com/stretchr/testify/require" "go.uber.org/zap/zaptest" ) @@ -65,6 +67,102 @@ func TestListObjectNullVersions(t *testing.T) { require.Equal(t, data.UnversionedObjectVersionID, result.Version[1].VersionID) } +func TestListObjectsWithOldTreeNodes(t *testing.T) { + hc := prepareHandlerContext(t) + + bktName, objName := "bucket-versioning-enabled", "object" + bktInfo := createTestBucket(hc, bktName) + + srcEnc, err := encryption.NewParams([]byte("1234567890qwertyuiopasdfghjklzxc")) + require.NoError(t, err) + objInfo := createTestObject(hc, bktInfo, objName, *srcEnc) + _ = objInfo + + prm := &tree.GetNodesParams{ + BktInfo: bktInfo, + TreeID: "version", + Path: []string{objName}, + LatestOnly: true, + AllAttrs: true, + } + nodes, err := hc.treeMock.GetNodes(hc.Context(), prm) + require.NoError(t, err) + require.Len(t, nodes, 1) + node := nodes[0] + meta := make(map[string]string, len(node.GetMeta())) + for _, m := range node.GetMeta() { + if m.GetKey() != "Created" && m.GetKey() != "Owner" { + meta[m.GetKey()] = string(m.GetValue()) + } + } + + err = hc.treeMock.MoveNode(hc.Context(), bktInfo, "version", node.GetNodeID(), node.GetParentID(), meta) + require.NoError(t, err) + + list := listObjectsV1(hc, bktName, "", "", "", -1) + require.Len(t, list.Contents, 1) + realSize, err := layer.GetObjectSize(objInfo) + require.NoError(t, err) + require.Equal(t, objInfo.Owner.EncodeToString(), list.Contents[0].Owner.ID) + require.Equal(t, objInfo.Created.UTC().Format(time.RFC3339), list.Contents[0].LastModified) + require.Equal(t, realSize, list.Contents[0].Size) +} + +func prepareObjects(hc *handlerContext, bktInfo *data.BucketInfo, prefix string, size int) []string { + treeID := "version" + parentID := uint64(0) + if prefix != "" { + for _, filename := range strings.Split(prefix, "/") { + nodeID, err := hc.treeMock.AddNode(hc.Context(), bktInfo, treeID, parentID, map[string]string{ + "FileName": filename, + }) + require.NoError(hc.t, err) + parentID = nodeID + } + prefix += "/" + } + + objects := make([]string, size) + + for i := range objects { + filename := "object" + strconv.Itoa(i) + filepath := prefix + filename + + prm := layer.PrmObjectCreate{ + Container: bktInfo.CID, + Filepath: filepath, + Payload: nil, + } + + id, err := hc.tp.CreateObject(hc.Context(), prm) + require.NoError(hc.t, err) + + newVersion := &data.NodeVersion{ + BaseNodeVersion: data.BaseNodeVersion{ + OID: id, + ETag: "12345678", + FilePath: filepath, + }, + IsUnversioned: true, + IsCombined: false, + } + + _, err = hc.treeMock.AddNodeBase(hc.Context(), bktInfo, treeID, parentID, map[string]string{ + "OID": newVersion.OID.EncodeToString(), + "FileName": filename, + "IsUnversioned": "true", + }, false) + require.NoError(hc.t, err) + objects[i] = filepath + } + + hc.treeMock.Sort() + + sort.Strings(objects) + + return objects +} + func TestListObjectsContextCanceled(t *testing.T) { layerCfg := layer.DefaultCachesConfigs(zaptest.NewLogger(t)) layerCfg.SessionList.Lifetime = time.Hour diff --git a/api/layer/listing.go b/api/layer/listing.go index 89ca9d9..f41b5e0 100644 --- a/api/layer/listing.go +++ b/api/layer/listing.go @@ -535,7 +535,13 @@ func (n *layer) initWorkerPool(ctx context.Context, size int, p commonVersionsLi } } - node.NodeVersion.FillExtra(oi) + realSize, err := GetObjectSize(oi) + if err != nil { + reqLog.Debug("failed to get real object size", zap.Error(err)) + realSize = oi.Size + } + + node.NodeVersion.FillExtra(&oi.Owner, &oi.Created, realSize) select { case <-ctx.Done():