From 9c058a70fdfc0d930a9bc80d5bff17fb50f29aa3 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 13 Aug 2021 17:13:14 +0300 Subject: [PATCH] [#122] Add tests Signed-off-by: Denis Kirillov --- api/handler/object_list.go | 12 +- api/layer/layer.go | 349 ++++++++++---------- api/layer/object.go | 248 ++++++++------ api/layer/util.go | 75 ++--- api/layer/versioning_test.go | 608 +++++++++++++++++++++++++++++++++++ 5 files changed, 960 insertions(+), 332 deletions(-) create mode 100644 api/layer/versioning_test.go diff --git a/api/handler/object_list.go b/api/handler/object_list.go index ef8f0dd4..d0233e98 100644 --- a/api/handler/object_list.go +++ b/api/handler/object_list.go @@ -276,7 +276,7 @@ func encodeListObjectVersionsToResponse(info *layer.ListObjectVersionsInfo, buck DisplayName: ver.Object.Owner.String(), }, Size: ver.Object.Size, - VersionID: ver.VersionID, + VersionID: ver.Object.Version(), ETag: ver.Object.HashSum, }) } @@ -284,13 +284,13 @@ func encodeListObjectVersionsToResponse(info *layer.ListObjectVersionsInfo, buck for _, del := range info.DeleteMarker { res.DeleteMarker = append(res.DeleteMarker, DeleteMarkerEntry{ IsLatest: del.IsLatest, - Key: del.Key, - LastModified: del.LastModified, + Key: del.Object.Name, + LastModified: del.Object.Created.Format(time.RFC3339), Owner: Owner{ - ID: del.Owner.String(), - DisplayName: del.Owner.String(), + ID: del.Object.Owner.String(), + DisplayName: del.Object.Owner.String(), }, - VersionID: del.VersionID, + VersionID: del.Object.Version(), }) } diff --git a/api/layer/layer.go b/api/layer/layer.go index 31773fec..fbd915fe 100644 --- a/api/layer/layer.go +++ b/api/layer/layer.go @@ -136,6 +136,13 @@ type ( VersionID string } + objectVersions struct { + objects []*ObjectInfo + addList []string + delList []string + isSorted bool + } + // NeoFS provides basic NeoFS interface. NeoFS interface { Get(ctx context.Context, address *object.Address) (*object.Object, error) @@ -170,9 +177,66 @@ type ( } ) +func (v *objectVersions) appendVersion(oi *ObjectInfo) { + addVers := append(splitVersions(oi.Headers[versionsAddAttr]), oi.Version()) + delVers := splitVersions(oi.Headers[versionsDelAttr]) + v.objects = append(v.objects, oi) + for _, add := range addVers { + if !contains(v.addList, add) { + v.addList = append(v.addList, add) + } + } + for _, del := range delVers { + if !contains(v.delList, del) { + v.delList = append(v.delList, del) + } + } + v.isSorted = false +} + +func (v *objectVersions) sort() { + if !v.isSorted { + sortVersions(v.objects) + v.isSorted = true + } +} + +func (v *objectVersions) getLast() *ObjectInfo { + if len(v.objects) == 0 { + return nil + } + + v.sort() + existedVersions := getExistedVersions(v) + for i := len(v.objects) - 1; i >= 0; i-- { + if contains(existedVersions, v.objects[i].Version()) { + delMarkHeader := v.objects[i].Headers[versionsDeleteMarkAttr] + if delMarkHeader == "" { + return v.objects[i] + } + if delMarkHeader == "*" { + return nil + } + } + } + + return nil +} + +func (v *objectVersions) getAddHeader() string { + return strings.Join(v.addList, ",") +} + +func (v *objectVersions) getDelHeader() string { + return strings.Join(v.delList, ",") +} + const ( - unversionedObjectVersionID = "null" - bktVersionSettingsObject = ".s3-versioning-settings" + unversionedObjectVersionID = "null" + bktVersionSettingsObject = ".s3-versioning-settings" + objectSystemAttributeName = "S3-System-name" + attrVersionsIgnore = "S3-Versions-ignore" + attrSettingsVersioningEnabled = "S3-Settings-Versioning-enabled" ) func (t *VersionedObject) String() string { @@ -322,16 +386,6 @@ func (n *layer) GetObject(ctx context.Context, p *GetObjectParams) error { return nil } -//func (n *layer) checkObject(ctx context.Context, cid *cid.ID, filename string) error { -// var err error -// -// if _, err = n.objectFindID(ctx, &findParams{cid: cid, val: filename}); err == nil { -// return new(errors.ObjectAlreadyExists) -// } -// -// return err -//} - // GetObjectInfo returns meta information about the object. func (n *layer) GetObjectInfo(ctx context.Context, p *HeadObjectParams) (*ObjectInfo, error) { bkt, err := n.GetBucketInfo(ctx, p.Bucket) @@ -341,20 +395,14 @@ func (n *layer) GetObjectInfo(ctx context.Context, p *HeadObjectParams) (*Object } if len(p.VersionID) == 0 { - objInfo, err := n.headLastVersion(ctx, bkt, p.Object) - if err == nil { - if deleteMark, err2 := strconv.ParseBool(objInfo.Headers[versionsDeleteMarkAttr]); err2 == nil && deleteMark { - return nil, errors.GetAPIError(errors.ErrNoSuchKey) - } - } - return objInfo, err + return n.headLastVersionIfNotDeleted(ctx, bkt, p.Object) } return n.headVersion(ctx, bkt, p.VersionID) } func (n *layer) getSettingsObjectInfo(ctx context.Context, bkt *BucketInfo) (*ObjectInfo, error) { - oid, err := n.objectFindID(ctx, &findParams{cid: bkt.CID, val: bktVersionSettingsObject}) + oid, err := n.objectFindID(ctx, &findParams{cid: bkt.CID, attr: objectSystemAttributeName, val: bktVersionSettingsObject}) if err != nil { return nil, err } @@ -428,73 +476,24 @@ func (n *layer) deleteObject(ctx context.Context, bkt *BucketInfo, obj *Versione } if versioningEnabled { + p := &PutObjectParams{ + Object: obj.Name, + Reader: bytes.NewReader(nil), + Header: map[string]string{versionsDeleteMarkAttr: obj.VersionID}, + } if len(obj.VersionID) != 0 { - id := object.NewID() - if err := id.Parse(obj.VersionID); err != nil { - return &errors.DeleteError{Err: errors.GetAPIError(errors.ErrInvalidVersion), Object: obj.String()} + id, err := n.checkVersionsExists(ctx, bkt, obj) + if err != nil { + return err } ids = []*object.ID{id} - lastObject, err := n.headLastVersion(ctx, bkt, obj.Name) - if err != nil { - return &errors.DeleteError{Err: err, Object: obj.String()} - } - if !strings.Contains(lastObject.Headers[versionsAddAttr], obj.VersionID) || - strings.Contains(lastObject.Headers[versionsDelAttr], obj.VersionID) { - return &errors.DeleteError{Err: errors.GetAPIError(errors.ErrInvalidVersion), Object: obj.String()} - } - - if lastObject.ID().String() == obj.VersionID { - if added := lastObject.Headers[versionsAddAttr]; len(added) > 0 { - addedVersions := strings.Split(added, ",") - sourceCopyVersion, err := n.headVersion(ctx, bkt, addedVersions[len(addedVersions)-1]) - if err != nil { - return &errors.DeleteError{Err: err, Object: obj.String()} - } - p := &CopyObjectParams{ - SrcObject: sourceCopyVersion, - DstBucket: bkt.Name, - DstObject: obj.Name, - SrcSize: sourceCopyVersion.Size, - Header: map[string]string{versionsDelAttr: obj.VersionID}, - } - if _, err := n.CopyObject(ctx, p); err != nil { - return err - } - } else { - p := &PutObjectParams{ - Object: obj.Name, - Reader: bytes.NewReader(nil), - Header: map[string]string{ - versionsDelAttr: obj.VersionID, - versionsDeleteMarkAttr: strconv.FormatBool(true), - }, - } - if _, err := n.objectPut(ctx, bkt, p); err != nil { - return &errors.DeleteError{Err: err, Object: obj.String()} - } - } - } else { - p := &CopyObjectParams{ - SrcObject: lastObject, - DstBucket: bkt.Name, - DstObject: obj.Name, - SrcSize: lastObject.Size, - Header: map[string]string{versionsDelAttr: obj.VersionID}, - } - if _, err := n.CopyObject(ctx, p); err != nil { - return err - } - } + p.Header[versionsDelAttr] = obj.VersionID } else { - p := &PutObjectParams{ - Object: obj.Name, - Reader: bytes.NewReader(nil), - Header: map[string]string{versionsDeleteMarkAttr: strconv.FormatBool(true)}, - } - if _, err := n.objectPut(ctx, bkt, p); err != nil { - return &errors.DeleteError{Err: err, Object: obj.String()} - } + p.Header[versionsDeleteMarkAttr] = "*" + } + if _, err = n.objectPut(ctx, bkt, p); err != nil { + return &errors.DeleteError{Err: err, Object: obj.String()} } } else { ids, err = n.objectSearch(ctx, &findParams{cid: bkt.CID, val: obj.Name}) @@ -512,6 +511,23 @@ func (n *layer) deleteObject(ctx context.Context, bkt *BucketInfo, obj *Versione return nil } +func (n *layer) checkVersionsExists(ctx context.Context, bkt *BucketInfo, obj *VersionedObject) (*object.ID, error) { + id := object.NewID() + if err := id.Parse(obj.VersionID); err != nil { + return nil, &errors.DeleteError{Err: errors.GetAPIError(errors.ErrInvalidVersion), Object: obj.String()} + } + + versions, err := n.headVersions(ctx, bkt, obj.Name) + if err != nil { + return nil, &errors.DeleteError{Err: err, Object: obj.String()} + } + if !contains(getExistedVersions(versions), obj.VersionID) { + return nil, &errors.DeleteError{Err: errors.GetAPIError(errors.ErrInvalidVersion), Object: obj.String()} + } + + return id, nil +} + // DeleteObjects from the storage. func (n *layer) DeleteObjects(ctx context.Context, bucket string, objects []*VersionedObject) []error { var errs = make([]error, 0, len(objects)) @@ -560,24 +576,17 @@ func (n *layer) DeleteBucket(ctx context.Context, p *DeleteBucketParams) error { } func (n *layer) ListObjectVersions(ctx context.Context, p *ListObjectVersionsParams) (*ListObjectVersionsInfo, error) { - var ( - res = ListObjectVersionsInfo{} - err error - bkt *BucketInfo - ids []*object.ID - latest = make(map[string]*ObjectVersionInfo) - ) + res := ListObjectVersionsInfo{} + versions := make(map[string]*objectVersions) - if bkt, err = n.GetBucketInfo(ctx, p.Bucket); err != nil { - return nil, err - } else if ids, err = n.objectSearch(ctx, &findParams{cid: bkt.CID}); err != nil { + bkt, err := n.GetBucketInfo(ctx, p.Bucket) + if err != nil { + return nil, err + } + ids, err := n.objectSearch(ctx, &findParams{cid: bkt.CID}) + if err != nil { return nil, err } - - versions := make([]*ObjectVersionInfo, 0, len(ids)) - deleted := make([]*DeletedObjectInfo, 0, len(ids)) - - deletedVersions := []string{} for _, id := range ids { meta, err := n.objectHead(ctx, bkt.CID, id) @@ -585,102 +594,80 @@ func (n *layer) ListObjectVersions(ctx context.Context, p *ListObjectVersionsPar n.log.Warn("could not fetch object meta", zap.Error(err)) continue } - if ov := objectVersionInfoFromMeta(bkt, meta, p.Prefix, p.Delimiter); ov != nil { - if ov.Object.Name <= p.KeyMarker { + if oi := objectInfoFromMeta(bkt, meta, p.Prefix, p.Delimiter); oi != nil { + if oi.Name <= p.KeyMarker { continue } - if currentLatest, ok := latest[ov.Object.Name]; ok { - if less(currentLatest, ov) { - latest[ov.Object.Name] = ov - } - } else { - latest[ov.Object.Name] = ov + if isSystem(oi) { + continue } - if del := ov.Object.Headers[versionsDelAttr]; len(del) != 0 { - deletedVersions = append(deletedVersions, strings.Split(del, ",")...) - } - - if parsed, err := strconv.ParseBool(ov.Object.Headers[versionsDeleteMarkAttr]); err == nil && parsed { - deleted = append(deleted, &DeletedObjectInfo{ - Owner: ov.Object.Owner, - Key: ov.Object.Name, - VersionID: ov.VersionID, - LastModified: ov.Object.Created.Format(time.RFC3339), - }) + if objVersions, ok := versions[oi.Name]; ok { + objVersions.appendVersion(oi) + versions[oi.Name] = objVersions } else { - versions = append(versions, ov) + objVersion := &objectVersions{} + objVersion.appendVersion(oi) + versions[oi.Name] = objVersion } } } - sort.Slice(versions, func(i, j int) bool { - if contains(deletedVersions, versions[i].VersionID) { - return true - } - if contains(deletedVersions, versions[j].VersionID) { - return false - } - if versions[i].Object.Name == versions[j].Object.Name { - if versions[i].CreationEpoch == versions[j].CreationEpoch { - return versions[i].VersionID < versions[j].VersionID - } - return versions[i].CreationEpoch < versions[j].CreationEpoch - } - return versions[i].Object.Name < versions[j].Object.Name + for _, v := range versions { + existed, deleted := triageVersions(v) + res.Version = append(res.Version, existed...) + res.DeleteMarker = append(res.DeleteMarker, deleted...) + } + + sort.Slice(res.Version, func(i, j int) bool { + return res.Version[i].Object.Name < res.Version[j].Object.Name + }) + sort.Slice(res.DeleteMarker, func(i, j int) bool { + return res.DeleteMarker[i].Object.Name < res.DeleteMarker[j].Object.Name }) - - for i, objVersion := range versions { - if i == len(versions)-1 || objVersion.Object.Name != versions[i+1].Object.Name { - objVersion.IsLatest = true - } - } - - for i, objVersion := range versions { - if !contains(deletedVersions, objVersion.VersionID) { - versions = versions[i:] - break - } - } - - for i, objVersion := range deleted { - if !contains(deletedVersions, objVersion.VersionID) { - deleted = deleted[i:] - break - } - } - - //if len(versions) > p.MaxKeys { - // res.IsTruncated = true - // - // lastVersion := versions[p.MaxKeys-1] - // res.KeyMarker = lastVersion.Object.Name - // res.VersionIDMarker = lastVersion.VersionID - // - // nextVersion := versions[p.MaxKeys] - // res.NextKeyMarker = nextVersion.Object.Name - // res.NextVersionIDMarker = nextVersion.VersionID - // - // versions = versions[:p.MaxKeys] - //} - // - //for _, ov := range versions { - // if isDir := uniqNames[ov.Object.Name]; isDir { - // res.CommonPrefixes = append(res.CommonPrefixes, &ov.Object.Name) - // } else { - // res.Version = append(res.Version, ov) - // } - //} - - res.Version = versions - res.DeleteMarker = deleted return &res, nil } -func less(ov1, ov2 *ObjectVersionInfo) bool { +func sortVersions(versions []*ObjectInfo) { + sort.Slice(versions, func(i, j int) bool { + return less(versions[i], versions[j]) + }) +} + +func triageVersions(objectVersions *objectVersions) ([]*ObjectVersionInfo, []*ObjectVersionInfo) { + if objectVersions == nil || len(objectVersions.objects) == 0 { + return nil, nil + } + + sortVersions(objectVersions.objects) + + var resVersion []*ObjectVersionInfo + var resDelMarkVersions []*ObjectVersionInfo + + isLatest := true + for i := len(objectVersions.objects) - 1; i >= 0; i-- { + version := objectVersions.objects[i] + if contains(objectVersions.delList, version.Version()) { + continue + } + + ovi := &ObjectVersionInfo{Object: version, IsLatest: isLatest} + isLatest = false + + if len(version.Headers[versionsDeleteMarkAttr]) == 0 { + resVersion = append(resVersion, ovi) + } else { + resDelMarkVersions = append(resDelMarkVersions, ovi) + } + } + + return resVersion, resDelMarkVersions +} + +func less(ov1, ov2 *ObjectInfo) bool { if ov1.CreationEpoch == ov2.CreationEpoch { - return ov1.VersionID < ov2.VersionID + return ov1.Version() < ov2.Version() } return ov1.CreationEpoch < ov2.CreationEpoch } @@ -711,7 +698,7 @@ func (n *layer) PutBucketVersioning(ctx context.Context, p *PutVersioningParams) attributes := make([]*object.Attribute, 0, 3) filename := object.NewAttribute() - filename.SetKey(object.AttributeFileName) + filename.SetKey(objectSystemAttributeName) filename.SetValue(bktVersionSettingsObject) createdAt := object.NewAttribute() @@ -719,11 +706,11 @@ func (n *layer) PutBucketVersioning(ctx context.Context, p *PutVersioningParams) createdAt.SetValue(strconv.FormatInt(time.Now().UTC().Unix(), 10)) versioningIgnore := object.NewAttribute() - versioningIgnore.SetKey("S3-Versions-ignore") + versioningIgnore.SetKey(attrVersionsIgnore) versioningIgnore.SetValue(strconv.FormatBool(true)) settingsVersioningEnabled := object.NewAttribute() - settingsVersioningEnabled.SetKey("S3-Settings-Versioning-enabled") + settingsVersioningEnabled.SetKey(attrSettingsVersioningEnabled) settingsVersioningEnabled.SetValue(strconv.FormatBool(p.Settings.VersioningEnabled)) attributes = append(attributes, filename, createdAt, versioningIgnore, settingsVersioningEnabled) diff --git a/api/layer/object.go b/api/layer/object.go index 0edf92b8..b77c3589 100644 --- a/api/layer/object.go +++ b/api/layer/object.go @@ -3,7 +3,6 @@ package layer import ( "context" "errors" - "fmt" "io" "net/url" "sort" @@ -14,14 +13,16 @@ import ( "github.com/nspcc-dev/neofs-api-go/pkg/client" cid "github.com/nspcc-dev/neofs-api-go/pkg/container/id" "github.com/nspcc-dev/neofs-api-go/pkg/object" + "github.com/nspcc-dev/neofs-api-go/pkg/owner" apiErrors "github.com/nspcc-dev/neofs-s3-gw/api/errors" "go.uber.org/zap" ) type ( findParams struct { - val string - cid *cid.ID + attr string + val string + cid *cid.ID } getParams struct { @@ -78,7 +79,11 @@ func (n *layer) objectSearch(ctx context.Context, p *findParams) ([]*object.ID, if filename, err := url.QueryUnescape(p.val); err != nil { return nil, err } else if filename != "" { - opts.AddFilter(object.AttributeFileName, filename, object.MatchStringEqual) + if p.attr == "" { + opts.AddFilter(object.AttributeFileName, filename, object.MatchStringEqual) + } else { + opts.AddFilter(p.attr, filename, object.MatchStringEqual) + } } return n.pool.SearchObject(ctx, new(client.SearchObjectParams).WithContainerID(p.cid).WithSearchFilters(opts), n.BearerOpt(ctx)) } @@ -123,60 +128,59 @@ func (n *layer) objectRange(ctx context.Context, p *getParams) ([]byte, error) { // objectPut into NeoFS, took payload from io.Reader. func (n *layer) objectPut(ctx context.Context, bkt *BucketInfo, p *PutObjectParams) (*ObjectInfo, error) { - var ( - err error - obj string - own = n.Owner(ctx) - ) - - if p.Object == bktVersionSettingsObject { - return nil, fmt.Errorf("trying put bucket settings object") - } - - if obj, err = url.QueryUnescape(p.Object); err != nil { + own := n.Owner(ctx) + obj, err := url.QueryUnescape(p.Object) + if err != nil { return nil, err } versioningEnabled := n.isVersioningEnabled(ctx, bkt) - lastVersionInfo, err := n.headLastVersion(ctx, bkt, p.Object) + versions, err := n.headVersions(ctx, bkt, obj) if err != nil && !apiErrors.IsS3Error(err, apiErrors.ErrNoSuchKey) { return nil, err } + idsToDeleteArr := updateCRDT2PSetHeaders(p, versions, versioningEnabled) - attributes := make([]*object.Attribute, 0, len(p.Header)+1) - var idsToDeleteArr []*object.ID - if lastVersionInfo != nil { - if versioningEnabled { - versionsAddedStr := lastVersionInfo.Headers[versionsAddAttr] - if len(versionsAddedStr) != 0 { - versionsAddedStr += "," - } - versionsAddedStr += lastVersionInfo.ID().String() - p.Header[versionsAddAttr] = versionsAddedStr + rawObject := formRawObject(p, bkt.CID, own, obj) + r := newDetector(p.Reader) - deleted := p.Header[versionsDelAttr] - if delVersions := lastVersionInfo.Headers[versionsDelAttr]; len(delVersions) != 0 { - if len(deleted) == 0 { - deleted = delVersions - } else { - deleted = delVersions + "," + deleted - } - } - if len(deleted) != 0 { - p.Header[versionsDelAttr] = deleted - } - } else { - versionsDeletedStr := lastVersionInfo.Headers[versionsDelAttr] - if len(versionsDeletedStr) != 0 { - versionsDeletedStr += "," - } - versionsDeletedStr += lastVersionInfo.ID().String() - p.Header[versionsDelAttr] = versionsDeletedStr + ops := new(client.PutObjectParams).WithObject(rawObject.Object()).WithPayloadReader(r) + oid, err := n.pool.PutObject(ctx, ops, n.BearerOpt(ctx)) + if err != nil { + return nil, err + } - idsToDeleteArr = append(idsToDeleteArr, lastVersionInfo.ID()) + meta, err := n.objectHead(ctx, bkt.CID, oid) + if err != nil { + return nil, err + } + + for _, id := range idsToDeleteArr { + if err = n.objectDelete(ctx, bkt.CID, id); err != nil { + n.log.Warn("couldn't delete object", + zap.Stringer("version id", id), + zap.Error(err)) } } + return &ObjectInfo{ + id: oid, + bucketID: bkt.CID, + + Owner: own, + Bucket: p.Bucket, + Name: p.Object, + Size: p.Size, + Created: time.Now(), + CreationEpoch: meta.CreationEpoch(), + Headers: p.Header, + ContentType: r.contentType, + HashSum: meta.PayloadChecksum().String(), + }, nil +} + +func formRawObject(p *PutObjectParams, bktID *cid.ID, own *owner.ID, obj string) *object.RawObject { + attributes := make([]*object.Attribute, 0, len(p.Header)+2) filename := object.NewAttribute() filename.SetKey(object.AttributeFileName) filename.SetValue(obj) @@ -197,55 +201,68 @@ func (n *layer) objectPut(ctx context.Context, bkt *BucketInfo, p *PutObjectPara raw := object.NewRaw() raw.SetOwnerID(own) - raw.SetContainerID(bkt.CID) + raw.SetContainerID(bktID) raw.SetAttributes(attributes...) - r := newDetector(p.Reader) + return raw +} - ops := new(client.PutObjectParams).WithObject(raw.Object()).WithPayloadReader(r) - oid, err := n.pool.PutObject( - ctx, - ops, - n.BearerOpt(ctx), - ) - if err != nil { - return nil, err +func updateCRDT2PSetHeaders(p *PutObjectParams, versions *objectVersions, versioningEnabled bool) []*object.ID { + var idsToDeleteArr []*object.ID + if versions == nil { + return idsToDeleteArr } - meta, err := n.objectHead(ctx, bkt.CID, oid) - if err != nil { - return nil, err - } + if versioningEnabled { + if len(versions.addList) != 0 { + p.Header[versionsAddAttr] = versions.getAddHeader() + } - if err = n.objCache.Put(addr, *meta); err != nil { - n.log.Error("couldn't cache an object", zap.Error(err)) - } + deleted := versions.getDelHeader() + // p.Header[versionsDelAttr] can be not empty when deleting specific version + if delAttr := p.Header[versionsDelAttr]; len(delAttr) != 0 { + if len(deleted) != 0 { + p.Header[versionsDelAttr] = deleted + "," + delAttr + } else { + p.Header[versionsDelAttr] = delAttr + } + } else if len(deleted) != 0 { + p.Header[versionsDelAttr] = deleted + } + } else { + versionsDeletedStr := versions.getDelHeader() + if len(versionsDeletedStr) != 0 { + versionsDeletedStr += "," + } - objInfo := &ObjectInfo{ - id: oid, + lastVersion := versions.getLast() + p.Header[versionsDelAttr] = versionsDeletedStr + lastVersion.Version() + idsToDeleteArr = append(idsToDeleteArr, lastVersion.ID()) - Owner: own, - Bucket: p.Bucket, - Name: p.Object, - Size: p.Size, - Created: time.Now(), - Headers: p.Header, - ContentType: r.contentType, - HashSum: meta.PayloadChecksum().String(), - } - - for _, id := range idsToDeleteArr { - if err = n.objectDelete(ctx, bkt.CID, id); err != nil { - n.log.Warn("couldn't delete object", - zap.Stringer("version id", id), - zap.Error(err)) + for _, version := range versions.objects { + if contains(versions.delList, version.Version()) { + idsToDeleteArr = append(idsToDeleteArr, version.ID()) + } } } - return objInfo, nil + return idsToDeleteArr } -func (n *layer) headLastVersion(ctx context.Context, bkt *BucketInfo, objectName string) (*ObjectInfo, error) { +func (n *layer) headLastVersionIfNotDeleted(ctx context.Context, bkt *BucketInfo, objectName string) (*ObjectInfo, error) { + versions, err := n.headVersions(ctx, bkt, objectName) + if err != nil { + return nil, err + } + + lastVersion := versions.getLast() + if lastVersion == nil { + return nil, apiErrors.GetAPIError(apiErrors.ErrNoSuchKey) + } + return lastVersion, nil +} + +func (n *layer) headVersions(ctx context.Context, bkt *BucketInfo, objectName string) (*objectVersions, error) { ids, err := n.objectSearch(ctx, &findParams{cid: bkt.CID, val: objectName}) if err != nil { return nil, err @@ -255,7 +272,7 @@ func (n *layer) headLastVersion(ctx context.Context, bkt *BucketInfo, objectName return nil, apiErrors.GetAPIError(apiErrors.ErrNoSuchKey) } - infos := make([]*object.Object, 0, len(ids)) + versions := &objectVersions{} for _, id := range ids { meta, err := n.objectHead(ctx, bkt.CID, id) if err != nil { @@ -265,14 +282,15 @@ func (n *layer) headLastVersion(ctx context.Context, bkt *BucketInfo, objectName zap.Error(err)) continue } - infos = append(infos, meta) + if oi := objectInfoFromMeta(bkt, meta, "", ""); oi != nil { + if isSystem(oi) { + continue + } + versions.appendVersion(oi) + } } - sort.Slice(infos, func(i, j int) bool { - return infos[i].CreationEpoch() < infos[j].CreationEpoch() || (infos[i].CreationEpoch() == infos[j].CreationEpoch() && infos[i].ID().String() < infos[j].ID().String()) - }) - - return objectInfoFromMeta(bkt, infos[len(infos)-1], "", ""), nil + return versions, nil } func (n *layer) headVersion(ctx context.Context, bkt *BucketInfo, versionID string) (*ObjectInfo, error) { @@ -378,17 +396,12 @@ func (n *layer) ListObjectsV2(ctx context.Context, p *ListObjectsParamsV2) (*Lis } func (n *layer) listSortedObjectsFromNeoFS(ctx context.Context, p allObjectParams) ([]*ObjectInfo, error) { - var ( - err error - ids []*object.ID - uniqNames = make(map[string]bool) - ) - - if ids, err = n.objectSearch(ctx, &findParams{cid: p.Bucket.CID}); err != nil { + ids, err := n.objectSearch(ctx, &findParams{cid: p.Bucket.CID}) + if err != nil { return nil, err } - objects := make([]*ObjectInfo, 0, len(ids)) + versions := make(map[string]*objectVersions, len(ids)/2) for _, id := range ids { meta, err := n.objectHead(ctx, p.Bucket.CID, id) @@ -397,14 +410,26 @@ func (n *layer) listSortedObjectsFromNeoFS(ctx context.Context, p allObjectParam continue } if oi := objectInfoFromMeta(p.Bucket, meta, p.Prefix, p.Delimiter); oi != nil { - // use only unique dir names - if _, ok := uniqNames[oi.Name]; ok { + if isSystem(oi) { continue } - uniqNames[oi.Name] = oi.isDir + if objVersions, ok := versions[oi.Name]; ok { + objVersions.appendVersion(oi) + versions[oi.Name] = objVersions + } else { + objVersion := &objectVersions{} + objVersion.appendVersion(oi) + versions[oi.Name] = objVersion + } + } + } - objects = append(objects, oi) + objects := make([]*ObjectInfo, 0, len(versions)) + for _, v := range versions { + lastVersion := v.getLast() + if lastVersion != nil { + objects = append(objects, lastVersion) } } @@ -415,6 +440,29 @@ func (n *layer) listSortedObjectsFromNeoFS(ctx context.Context, p allObjectParam return objects, nil } +func getExistedVersions(versions *objectVersions) []string { + var res []string + for _, add := range versions.addList { + if !contains(versions.delList, add) { + res = append(res, add) + } + } + return res +} + +func splitVersions(header string) []string { + if len(header) == 0 { + return nil + } + + return strings.Split(header, ",") +} + +func isSystem(obj *ObjectInfo) bool { + return len(obj.Headers[objectSystemAttributeName]) > 0 || + len(obj.Headers[attrVersionsIgnore]) > 0 +} + func trimAfterObjectName(startAfter string, objects []*ObjectInfo) []*ObjectInfo { if len(objects) != 0 && objects[len(objects)-1].Name <= startAfter { return nil diff --git a/api/layer/util.go b/api/layer/util.go index 6fce6919..a915c5d3 100644 --- a/api/layer/util.go +++ b/api/layer/util.go @@ -9,7 +9,6 @@ import ( "time" cid "github.com/nspcc-dev/neofs-api-go/pkg/container/id" - "github.com/nspcc-dev/neofs-api-go/pkg/object" "github.com/nspcc-dev/neofs-api-go/pkg/owner" "github.com/nspcc-dev/neofs-s3-gw/api" @@ -19,18 +18,19 @@ import ( type ( // ObjectInfo holds S3 object data. ObjectInfo struct { - id *object.ID - isDir bool + id *object.ID + bucketID *cid.ID + isDir bool - Bucket string - bucketID *cid.ID - Name string - Size int64 - ContentType string - Created time.Time - HashSum string - Owner *owner.ID - Headers map[string]string + Bucket string + Name string + Size int64 + ContentType string + Created time.Time + CreationEpoch uint64 + HashSum string + Owner *owner.ID + Headers map[string]string } // ListObjectsInfo contains common fields of data for ListObjectsV1 and ListObjectsV2. @@ -54,19 +54,8 @@ type ( // ObjectVersionInfo stores info about objects versions. ObjectVersionInfo struct { - Object *ObjectInfo - IsLatest bool - VersionID string - CreationEpoch uint64 - } - - // DeletedObjectInfo stores info about deleted versions of objects. - DeletedObjectInfo struct { - Owner *owner.ID - Key string - VersionID string - IsLatest bool - LastModified string + Object *ObjectInfo + IsLatest bool } // ListObjectVersionsInfo stores info and list of objects' versions. @@ -77,7 +66,7 @@ type ( NextKeyMarker string NextVersionIDMarker string Version []*ObjectVersionInfo - DeleteMarker []*DeletedObjectInfo + DeleteMarker []*ObjectVersionInfo VersionIDMarker string } ) @@ -137,29 +126,22 @@ func objectInfoFromMeta(bkt *BucketInfo, meta *object.Object, prefix, delimiter } return &ObjectInfo{ - id: meta.ID(), - isDir: isDir, + id: meta.ID(), + bucketID: bkt.CID, + isDir: isDir, - Bucket: bkt.Name, - bucketID: bkt.CID, - Name: filename, - Created: creation, - ContentType: mimeType, - Headers: userHeaders, - Owner: meta.OwnerID(), - Size: size, - HashSum: meta.PayloadChecksum().String(), + Bucket: bkt.Name, + Name: filename, + Created: creation, + CreationEpoch: meta.CreationEpoch(), + ContentType: mimeType, + Headers: userHeaders, + Owner: meta.OwnerID(), + Size: size, + HashSum: meta.PayloadChecksum().String(), } } -func objectVersionInfoFromMeta(bkt *BucketInfo, meta *object.Object, prefix, delimiter string) *ObjectVersionInfo { - oi := objectInfoFromMeta(bkt, meta, prefix, delimiter) - if oi == nil { - return nil - } - return &ObjectVersionInfo{Object: oi, VersionID: meta.ID().String(), CreationEpoch: meta.CreationEpoch()} -} - func filenameFromObject(o *object.Object) string { var name = o.ID().String() for _, attr := range o.Attributes() { @@ -179,6 +161,9 @@ func NameFromString(name string) (string, string) { // ID returns object ID from ObjectInfo. func (o *ObjectInfo) ID() *object.ID { return o.id } +// Version returns object version from ObjectInfo. +func (o *ObjectInfo) Version() string { return o.id.String() } + // CID returns bucket ID from ObjectInfo. func (o *ObjectInfo) CID() *cid.ID { return o.bucketID } diff --git a/api/layer/versioning_test.go b/api/layer/versioning_test.go new file mode 100644 index 00000000..31ec3e04 --- /dev/null +++ b/api/layer/versioning_test.go @@ -0,0 +1,608 @@ +package layer + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/sha256" + "fmt" + "io" + "strings" + "testing" + + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "github.com/nspcc-dev/neofs-api-go/pkg/acl/eacl" + "github.com/nspcc-dev/neofs-api-go/pkg/client" + "github.com/nspcc-dev/neofs-api-go/pkg/container" + cid "github.com/nspcc-dev/neofs-api-go/pkg/container/id" + "github.com/nspcc-dev/neofs-api-go/pkg/object" + "github.com/nspcc-dev/neofs-api-go/pkg/owner" + "github.com/nspcc-dev/neofs-api-go/pkg/session" + "github.com/nspcc-dev/neofs-api-go/pkg/token" + "github.com/nspcc-dev/neofs-s3-gw/api" + "github.com/nspcc-dev/neofs-s3-gw/creds/accessbox" + "github.com/nspcc-dev/neofs-sdk-go/pkg/logger" + "github.com/nspcc-dev/neofs-sdk-go/pkg/pool" + "github.com/stretchr/testify/require" +) + +type testPool struct { + objects map[string]*object.Object + containers map[string]*container.Container + currentEpoch uint64 +} + +func newTestPool() *testPool { + return &testPool{ + objects: make(map[string]*object.Object), + containers: make(map[string]*container.Container), + } +} + +func (t *testPool) PutObject(ctx context.Context, params *client.PutObjectParams, option ...client.CallOption) (*object.ID, error) { + b := make([]byte, 32) + if _, err := io.ReadFull(rand.Reader, b); err != nil { + return nil, err + } + + oid := object.NewID() + oid.SetSHA256(sha256.Sum256(b)) + + raw := object.NewRawFrom(params.Object()) + raw.SetID(oid) + raw.SetCreationEpoch(t.currentEpoch) + t.currentEpoch++ + + if params.PayloadReader() != nil { + all, err := io.ReadAll(params.PayloadReader()) + if err != nil { + return nil, err + } + raw.SetPayload(all) + } + + addr := object.NewAddress() + addr.SetObjectID(raw.ID()) + addr.SetContainerID(raw.ContainerID()) + + t.objects[addr.String()] = raw.Object() + return raw.ID(), nil +} + +func (t *testPool) DeleteObject(ctx context.Context, params *client.DeleteObjectParams, option ...client.CallOption) error { + delete(t.objects, params.Address().String()) + return nil +} + +func (t *testPool) GetObject(ctx context.Context, params *client.GetObjectParams, option ...client.CallOption) (*object.Object, error) { + if obj, ok := t.objects[params.Address().String()]; ok { + if params.PayloadWriter() != nil { + _, err := params.PayloadWriter().Write(obj.Payload()) + if err != nil { + return nil, err + } + } + return obj, nil + } + + return nil, fmt.Errorf("object not found " + params.Address().String()) +} + +func (t *testPool) GetObjectHeader(ctx context.Context, params *client.ObjectHeaderParams, option ...client.CallOption) (*object.Object, error) { + p := new(client.GetObjectParams).WithAddress(params.Address()) + return t.GetObject(ctx, p) +} + +func (t *testPool) ObjectPayloadRangeData(ctx context.Context, params *client.RangeDataParams, option ...client.CallOption) ([]byte, error) { + panic("implement me") +} + +func (t *testPool) ObjectPayloadRangeSHA256(ctx context.Context, params *client.RangeChecksumParams, option ...client.CallOption) ([][32]byte, error) { + panic("implement me") +} + +func (t *testPool) ObjectPayloadRangeTZ(ctx context.Context, params *client.RangeChecksumParams, option ...client.CallOption) ([][64]byte, error) { + panic("implement me") +} + +func (t *testPool) SearchObject(ctx context.Context, params *client.SearchObjectParams, option ...client.CallOption) ([]*object.ID, error) { + cidStr := params.ContainerID().String() + + var res []*object.ID + + if len(params.SearchFilters()) == 1 { + for k, v := range t.objects { + if strings.Contains(k, cidStr) { + res = append(res, v.ID()) + } + } + return res, nil + } + + filter := params.SearchFilters()[1] + if len(params.SearchFilters()) != 2 || filter.Operation() != object.MatchStringEqual || + (filter.Header() != object.AttributeFileName && filter.Header() != objectSystemAttributeName) { + return nil, fmt.Errorf("usupported filters") + } + + for k, v := range t.objects { + if strings.Contains(k, cidStr) && isMatched(v.Attributes(), filter) { + res = append(res, v.ID()) + } + } + + return res, nil +} + +func isMatched(attributes []*object.Attribute, filter object.SearchFilter) bool { + for _, attr := range attributes { + if attr.Key() == filter.Header() && attr.Value() == filter.Value() { + return true + } + } + + return false +} + +func (t *testPool) PutContainer(ctx context.Context, container *container.Container, option ...client.CallOption) (*cid.ID, error) { + b := make([]byte, 32) + if _, err := io.ReadFull(rand.Reader, b); err != nil { + return nil, err + } + + id := cid.New() + id.SetSHA256(sha256.Sum256(b)) + t.containers[id.String()] = container + + return id, nil +} + +func (t *testPool) GetContainer(ctx context.Context, id *cid.ID, option ...client.CallOption) (*container.Container, error) { + for k, v := range t.containers { + if k == id.String() { + return v, nil + } + } + + return nil, fmt.Errorf("container not found " + id.String()) +} + +func (t *testPool) ListContainers(ctx context.Context, id *owner.ID, option ...client.CallOption) ([]*cid.ID, error) { + var res []*cid.ID + for k := range t.containers { + cID := cid.New() + if err := cID.Parse(k); err != nil { + return nil, err + } + res = append(res, cID) + } + + return res, nil +} + +func (t *testPool) DeleteContainer(ctx context.Context, id *cid.ID, option ...client.CallOption) error { + delete(t.containers, id.String()) + return nil +} + +func (t *testPool) GetEACL(ctx context.Context, id *cid.ID, option ...client.CallOption) (*client.EACLWithSignature, error) { + panic("implement me") +} + +func (t *testPool) SetEACL(ctx context.Context, table *eacl.Table, option ...client.CallOption) error { + panic("implement me") +} + +func (t *testPool) AnnounceContainerUsedSpace(ctx context.Context, announcements []container.UsedSpaceAnnouncement, option ...client.CallOption) error { + panic("implement me") +} + +func (t *testPool) Connection() (client.Client, *session.Token, error) { + panic("implement me") +} + +func (t *testPool) OwnerID() *owner.ID { + return nil +} + +func (t *testPool) WaitForContainerPresence(ctx context.Context, id *cid.ID, params *pool.ContainerPollingParams) error { + return nil +} + +func (tc *testContext) putObject(content []byte) *ObjectInfo { + objInfo, err := tc.layer.PutObject(tc.ctx, &PutObjectParams{ + Bucket: tc.bkt, + Object: tc.obj, + Size: int64(len(content)), + Reader: bytes.NewReader(content), + Header: make(map[string]string), + }) + require.NoError(tc.t, err) + + return objInfo +} + +func (tc *testContext) getObject(objectName, versionID string, needError bool) (*ObjectInfo, []byte) { + objInfo, err := tc.layer.GetObjectInfo(tc.ctx, &HeadObjectParams{ + Bucket: tc.bkt, + Object: objectName, + VersionID: versionID, + }) + if needError { + require.Error(tc.t, err) + return nil, nil + } + require.NoError(tc.t, err) + + content := bytes.NewBuffer(nil) + err = tc.layer.GetObject(tc.ctx, &GetObjectParams{ + ObjectInfo: objInfo, + Writer: content, + VersionID: versionID, + }) + require.NoError(tc.t, err) + + return objInfo, content.Bytes() +} + +func (tc *testContext) deleteObject(objectName, versionID string) { + errs := tc.layer.DeleteObjects(tc.ctx, tc.bkt, []*VersionedObject{ + {Name: objectName, VersionID: versionID}, + }) + for _, err := range errs { + require.NoError(tc.t, err) + } +} + +func (tc *testContext) listObjectsV1() []*ObjectInfo { + res, err := tc.layer.ListObjectsV1(tc.ctx, &ListObjectsParamsV1{ + ListObjectsParamsCommon: ListObjectsParamsCommon{ + Bucket: tc.bkt, + MaxKeys: 1000, + }, + }) + require.NoError(tc.t, err) + return res.Objects +} + +func (tc *testContext) listObjectsV2() []*ObjectInfo { + res, err := tc.layer.ListObjectsV2(tc.ctx, &ListObjectsParamsV2{ + ListObjectsParamsCommon: ListObjectsParamsCommon{ + Bucket: tc.bkt, + MaxKeys: 1000, + }, + }) + require.NoError(tc.t, err) + return res.Objects +} + +func (tc *testContext) listVersions() *ListObjectVersionsInfo { + res, err := tc.layer.ListObjectVersions(tc.ctx, &ListObjectVersionsParams{ + Bucket: tc.bkt, + MaxKeys: 1000, + }) + require.NoError(tc.t, err) + return res +} + +func (tc *testContext) checkListObjects(ids ...*object.ID) { + objs := tc.listObjectsV1() + require.Equal(tc.t, len(ids), len(objs)) + for _, id := range ids { + require.Contains(tc.t, ids, id) + } + + objs = tc.listObjectsV2() + require.Equal(tc.t, len(ids), len(objs)) + for _, id := range ids { + require.Contains(tc.t, ids, id) + } +} + +type testContext struct { + t *testing.T + ctx context.Context + layer Client + bkt string + bktID *cid.ID + obj string + testPool *testPool +} + +func prepareContext(t *testing.T) *testContext { + key, err := keys.NewPrivateKey() + require.NoError(t, err) + + ctx := context.WithValue(context.Background(), api.BoxData, &accessbox.Box{ + Gate: &accessbox.GateData{ + BearerToken: token.NewBearerToken(), + GateKey: key.PublicKey(), + }, + }) + l, err := logger.New(logger.WithTraceLevel("panic")) + require.NoError(t, err) + tp := newTestPool() + + bktName := "testbucket1" + cnr := container.New(container.WithAttribute(container.AttributeName, bktName)) + bktID, err := tp.PutContainer(ctx, cnr) + require.NoError(t, err) + + return &testContext{ + ctx: ctx, + layer: NewLayer(l, tp), + bkt: bktName, + bktID: bktID, + obj: "obj1", + t: t, + testPool: tp, + } +} + +func TestSimpleVersioning(t *testing.T) { + tc := prepareContext(t) + _, err := tc.layer.PutBucketVersioning(tc.ctx, &PutVersioningParams{ + Bucket: tc.bkt, + Settings: &BucketSettings{VersioningEnabled: true}, + }) + require.NoError(t, err) + + obj1Content1 := []byte("content obj1 v1") + obj1v1 := tc.putObject(obj1Content1) + + obj1Content2 := []byte("content obj1 v2") + obj1v2 := tc.putObject(obj1Content2) + + objv2, buffer2 := tc.getObject(tc.obj, "", false) + require.Equal(t, obj1Content2, buffer2) + require.Contains(t, objv2.Headers[versionsAddAttr], obj1v1.ID().String()) + + _, buffer1 := tc.getObject(tc.obj, obj1v1.ID().String(), false) + require.Equal(t, obj1Content1, buffer1) + + tc.checkListObjects(obj1v2.ID()) +} + +func TestSimpleNoVersioning(t *testing.T) { + tc := prepareContext(t) + + obj1Content1 := []byte("content obj1 v1") + obj1v1 := tc.putObject(obj1Content1) + + obj1Content2 := []byte("content obj1 v2") + obj1v2 := tc.putObject(obj1Content2) + + objv2, buffer2 := tc.getObject(tc.obj, "", false) + require.Equal(t, obj1Content2, buffer2) + require.Contains(t, objv2.Headers[versionsDelAttr], obj1v1.ID().String()) + + tc.getObject(tc.obj, obj1v1.ID().String(), true) + tc.checkListObjects(obj1v2.ID()) +} + +func TestVersioningDeleteObject(t *testing.T) { + tc := prepareContext(t) + _, err := tc.layer.PutBucketVersioning(tc.ctx, &PutVersioningParams{ + Bucket: tc.bkt, + Settings: &BucketSettings{VersioningEnabled: true}, + }) + require.NoError(t, err) + + tc.putObject([]byte("content obj1 v1")) + tc.putObject([]byte("content obj1 v2")) + + tc.deleteObject(tc.obj, "") + tc.getObject(tc.obj, "", true) + + tc.checkListObjects() +} + +func TestVersioningDeleteSpecificObjectVersion(t *testing.T) { + tc := prepareContext(t) + _, err := tc.layer.PutBucketVersioning(tc.ctx, &PutVersioningParams{ + Bucket: tc.bkt, + Settings: &BucketSettings{VersioningEnabled: true}, + }) + require.NoError(t, err) + + tc.putObject([]byte("content obj1 v1")) + objV2Info := tc.putObject([]byte("content obj1 v2")) + objV3Content := []byte("content obj1 v3") + objV3Info := tc.putObject(objV3Content) + + tc.deleteObject(tc.obj, objV2Info.Version()) + tc.getObject(tc.obj, objV2Info.Version(), true) + + _, buffer3 := tc.getObject(tc.obj, "", false) + require.Equal(t, objV3Content, buffer3) + + tc.deleteObject(tc.obj, "") + tc.getObject(tc.obj, "", true) + + for _, ver := range tc.listVersions().DeleteMarker { + if ver.IsLatest { + tc.deleteObject(tc.obj, ver.Object.Version()) + } + } + + resInfo, buffer := tc.getObject(tc.obj, "", false) + require.Equal(t, objV3Content, buffer) + require.Equal(t, objV3Info.Version(), resInfo.Version()) +} + +func TestNoVersioningDeleteObject(t *testing.T) { + tc := prepareContext(t) + + tc.putObject([]byte("content obj1 v1")) + tc.putObject([]byte("content obj1 v2")) + + tc.deleteObject(tc.obj, "") + tc.getObject(tc.obj, "", true) + tc.checkListObjects() +} + +func TestGetLastVersion(t *testing.T) { + obj1 := getTestObjectInfo(1, getOID(1), "", "", "") + obj1V2 := getTestObjectInfo(1, getOID(2), "", "", "") + obj2 := getTestObjectInfo(2, getOID(2), obj1.Version(), "", "") + obj3 := getTestObjectInfo(3, getOID(3), joinVers(obj1, obj2), "", "*") + obj4 := getTestObjectInfo(4, getOID(4), joinVers(obj1, obj2), obj2.Version(), obj2.Version()) + obj5 := getTestObjectInfo(5, getOID(5), obj1.Version(), obj1.Version(), obj1.Version()) + obj6 := getTestObjectInfo(6, getOID(6), joinVers(obj1, obj2, obj3), obj3.Version(), obj3.Version()) + + for _, tc := range []struct { + versions *objectVersions + expected *ObjectInfo + }{ + { + versions: &objectVersions{}, + expected: nil, + }, + { + versions: &objectVersions{ + objects: []*ObjectInfo{obj2, obj1}, + addList: []string{obj1.Version(), obj2.Version()}, + }, + expected: obj2, + }, + { + versions: &objectVersions{ + objects: []*ObjectInfo{obj2, obj1, obj3}, + addList: []string{obj1.Version(), obj2.Version(), obj3.Version()}, + }, + expected: nil, + }, + { + versions: &objectVersions{ + objects: []*ObjectInfo{obj2, obj1, obj4}, + addList: []string{obj1.Version(), obj2.Version(), obj4.Version()}, + delList: []string{obj2.Version()}, + }, + expected: obj1, + }, + { + versions: &objectVersions{ + objects: []*ObjectInfo{obj1, obj5}, + addList: []string{obj1.Version(), obj5.Version()}, + delList: []string{obj1.Version()}, + }, + expected: nil, + }, + { + versions: &objectVersions{ + objects: []*ObjectInfo{obj5}, + }, + expected: nil, + }, + { + versions: &objectVersions{ + objects: []*ObjectInfo{obj1, obj2, obj3, obj6}, + addList: []string{obj1.Version(), obj2.Version(), obj3.Version(), obj6.Version()}, + delList: []string{obj3.Version()}, + }, + expected: obj2, + }, + { + versions: &objectVersions{ + objects: []*ObjectInfo{obj1, obj1V2}, + addList: []string{obj1.Version(), obj1V2.Version()}, + }, + // creation epochs are equal + // obj1 version/oid > obj1_1 version/oid + expected: obj1, + }, + } { + actualObjInfo := tc.versions.getLast() + require.Equal(t, tc.expected, actualObjInfo) + } +} + +func TestAppendVersions(t *testing.T) { + obj1 := getTestObjectInfo(1, getOID(1), "", "", "") + obj2 := getTestObjectInfo(2, getOID(2), obj1.Version(), "", "") + obj3 := getTestObjectInfo(3, getOID(3), joinVers(obj1, obj2), "", "*") + obj4 := getTestObjectInfo(4, getOID(4), joinVers(obj1, obj2), obj2.Version(), obj2.Version()) + + for _, tc := range []struct { + versions *objectVersions + objectToAdd *ObjectInfo + expectedVersions *objectVersions + }{ + { + versions: &objectVersions{}, + objectToAdd: obj1, + expectedVersions: &objectVersions{ + objects: []*ObjectInfo{obj1}, + addList: []string{obj1.Version()}, + }, + }, + { + versions: &objectVersions{objects: []*ObjectInfo{obj1}}, + objectToAdd: obj2, + expectedVersions: &objectVersions{ + objects: []*ObjectInfo{obj1, obj2}, + addList: []string{obj1.Version(), obj2.Version()}, + }, + }, + { + versions: &objectVersions{objects: []*ObjectInfo{obj1, obj2}}, + objectToAdd: obj3, + expectedVersions: &objectVersions{ + objects: []*ObjectInfo{obj1, obj2, obj3}, + addList: []string{obj1.Version(), obj2.Version(), obj3.Version()}, + }, + }, + { + versions: &objectVersions{objects: []*ObjectInfo{obj1, obj2}}, + objectToAdd: obj4, + expectedVersions: &objectVersions{ + objects: []*ObjectInfo{obj1, obj2, obj4}, + addList: []string{obj1.Version(), obj2.Version(), obj4.Version()}, + delList: []string{obj2.Version()}, + }, + }, + } { + tc.versions.appendVersion(tc.objectToAdd) + require.Equal(t, tc.expectedVersions, tc.versions) + } +} + +func joinVers(objs ...*ObjectInfo) string { + if len(objs) == 0 { + return "" + } + + var versions []string + for _, obj := range objs { + versions = append(versions, obj.Version()) + } + + return strings.Join(versions, ",") +} + +func getOID(id byte) *object.ID { + b := make([]byte, 32) + b[0] = id + oid := object.NewID() + oid.SetSHA256(sha256.Sum256(b)) + return oid +} + +func getTestObjectInfo(epoch uint64, oid *object.ID, addAttr, delAttr, delMarkAttr string) *ObjectInfo { + headers := make(map[string]string) + if addAttr != "" { + headers[versionsAddAttr] = addAttr + } + if delAttr != "" { + headers[versionsDelAttr] = delAttr + } + if delMarkAttr != "" { + headers[versionsDeleteMarkAttr] = delMarkAttr + } + + return &ObjectInfo{ + id: oid, + CreationEpoch: epoch, + Headers: headers, + } +}