r.loginov/frostfs-s3-gw
1
0
Fork 0
forked from TrueCloudLab/frostfs-s3-gw
Code Pull requests Activity

[#122] Add tests

Browse source 
Signed-off-by: Denis Kirillov <denis@nspcc.ru>
This commit is contained in:
Denis Kirillov 2021-08-13 17:13:14 +03:00
parent 43185de52a
commit 9c058a70fd
5 changed files with 960 additions and 332 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
api/handler/object_list.go
View file

@ -276,7 +276,7 @@ func encodeListObjectVersionsToResponse(info *layer.ListObjectVersionsInfo, buck
DisplayName: ver.Object.Owner.String(), DisplayName: ver.Object.Owner.String(),
}, },
Size: ver.Object.Size, Size: ver.Object.Size,
VersionID: ver.VersionID, VersionID: ver.Object.Version(),
ETag: ver.Object.HashSum, ETag: ver.Object.HashSum,
}) })
} }
@ -284,13 +284,13 @@ func encodeListObjectVersionsToResponse(info *layer.ListObjectVersionsInfo, buck
for _, del := range info.DeleteMarker { for _, del := range info.DeleteMarker {
res.DeleteMarker = append(res.DeleteMarker, DeleteMarkerEntry{ res.DeleteMarker = append(res.DeleteMarker, DeleteMarkerEntry{
IsLatest: del.IsLatest, IsLatest: del.IsLatest,
Key: del.Key, Key: del.Object.Name,
LastModified: del.LastModified, LastModified: del.Object.Created.Format(time.RFC3339),
Owner: Owner{ Owner: Owner{
ID: del.Owner.String(), ID: del.Object.Owner.String(),
DisplayName: del.Owner.String(), DisplayName: del.Object.Owner.String(),
}, },
VersionID: del.VersionID, VersionID: del.Object.Version(),
}) })
} }

349
api/layer/layer.go
View file

@ -136,6 +136,13 @@ type (
VersionID string VersionID string
} }
objectVersions struct {
objects []*ObjectInfo
addList []string
delList []string
isSorted bool
}
// NeoFS provides basic NeoFS interface. // NeoFS provides basic NeoFS interface.
NeoFS interface { NeoFS interface {
Get(ctx context.Context, address *object.Address) (*object.Object, error) Get(ctx context.Context, address *object.Address) (*object.Object, error)
@ -170,9 +177,66 @@ type (
} }
) )
func (v *objectVersions) appendVersion(oi *ObjectInfo) {
addVers := append(splitVersions(oi.Headers[versionsAddAttr]), oi.Version())
delVers := splitVersions(oi.Headers[versionsDelAttr])
v.objects = append(v.objects, oi)
for _, add := range addVers {
if !contains(v.addList, add) {
v.addList = append(v.addList, add)
}
}
for _, del := range delVers {
if !contains(v.delList, del) {
v.delList = append(v.delList, del)
}
}
v.isSorted = false
}
func (v *objectVersions) sort() {
if !v.isSorted {
sortVersions(v.objects)
v.isSorted = true
}
}
func (v *objectVersions) getLast() *ObjectInfo {
if len(v.objects) == 0 {
return nil
}
v.sort()
existedVersions := getExistedVersions(v)
for i := len(v.objects) - 1; i >= 0; i-- {
if contains(existedVersions, v.objects[i].Version()) {
delMarkHeader := v.objects[i].Headers[versionsDeleteMarkAttr]
if delMarkHeader == "" {
return v.objects[i]
}
if delMarkHeader == "*" {
return nil
}
}
}
return nil
}
func (v *objectVersions) getAddHeader() string {
return strings.Join(v.addList, ",")
}
func (v *objectVersions) getDelHeader() string {
return strings.Join(v.delList, ",")
}
const ( const (
unversionedObjectVersionID = "null" unversionedObjectVersionID = "null"
bktVersionSettingsObject = ".s3-versioning-settings" bktVersionSettingsObject = ".s3-versioning-settings"
objectSystemAttributeName = "S3-System-name"
attrVersionsIgnore = "S3-Versions-ignore"
attrSettingsVersioningEnabled = "S3-Settings-Versioning-enabled"
) )
func (t *VersionedObject) String() string { func (t *VersionedObject) String() string {
@ -322,16 +386,6 @@ func (n *layer) GetObject(ctx context.Context, p *GetObjectParams) error {
return nil return nil
} }
//func (n *layer) checkObject(ctx context.Context, cid *cid.ID, filename string) error {
// var err error
//
// if _, err = n.objectFindID(ctx, &findParams{cid: cid, val: filename}); err == nil {
// return new(errors.ObjectAlreadyExists)
// }
//
// return err
//}
// GetObjectInfo returns meta information about the object. // GetObjectInfo returns meta information about the object.
func (n *layer) GetObjectInfo(ctx context.Context, p *HeadObjectParams) (*ObjectInfo, error) { func (n *layer) GetObjectInfo(ctx context.Context, p *HeadObjectParams) (*ObjectInfo, error) {
bkt, err := n.GetBucketInfo(ctx, p.Bucket) bkt, err := n.GetBucketInfo(ctx, p.Bucket)
@ -341,20 +395,14 @@ func (n *layer) GetObjectInfo(ctx context.Context, p *HeadObjectParams) (*Object
} }
if len(p.VersionID) == 0 { if len(p.VersionID) == 0 {
objInfo, err := n.headLastVersion(ctx, bkt, p.Object) return n.headLastVersionIfNotDeleted(ctx, bkt, p.Object)
if err == nil {
if deleteMark, err2 := strconv.ParseBool(objInfo.Headers[versionsDeleteMarkAttr]); err2 == nil && deleteMark {
return nil, errors.GetAPIError(errors.ErrNoSuchKey)
}
}
return objInfo, err
} }
return n.headVersion(ctx, bkt, p.VersionID) return n.headVersion(ctx, bkt, p.VersionID)
} }
func (n *layer) getSettingsObjectInfo(ctx context.Context, bkt *BucketInfo) (*ObjectInfo, error) { func (n *layer) getSettingsObjectInfo(ctx context.Context, bkt *BucketInfo) (*ObjectInfo, error) {
oid, err := n.objectFindID(ctx, &findParams{cid: bkt.CID, val: bktVersionSettingsObject}) oid, err := n.objectFindID(ctx, &findParams{cid: bkt.CID, attr: objectSystemAttributeName, val: bktVersionSettingsObject})
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -428,73 +476,24 @@ func (n *layer) deleteObject(ctx context.Context, bkt *BucketInfo, obj *Versione
} }
if versioningEnabled { if versioningEnabled {
p := &PutObjectParams{
Object: obj.Name,
Reader: bytes.NewReader(nil),
Header: map[string]string{versionsDeleteMarkAttr: obj.VersionID},
}
if len(obj.VersionID) != 0 { if len(obj.VersionID) != 0 {
id := object.NewID() id, err := n.checkVersionsExists(ctx, bkt, obj)
if err := id.Parse(obj.VersionID); err != nil { if err != nil {
return &errors.DeleteError{Err: errors.GetAPIError(errors.ErrInvalidVersion), Object: obj.String()} return err
} }
ids = []*object.ID{id} ids = []*object.ID{id}
lastObject, err := n.headLastVersion(ctx, bkt, obj.Name) p.Header[versionsDelAttr] = obj.VersionID
if err != nil {
return &errors.DeleteError{Err: err, Object: obj.String()}
}
if !strings.Contains(lastObject.Headers[versionsAddAttr], obj.VersionID) ||
strings.Contains(lastObject.Headers[versionsDelAttr], obj.VersionID) {
return &errors.DeleteError{Err: errors.GetAPIError(errors.ErrInvalidVersion), Object: obj.String()}
}
if lastObject.ID().String() == obj.VersionID {
if added := lastObject.Headers[versionsAddAttr]; len(added) > 0 {
addedVersions := strings.Split(added, ",")
sourceCopyVersion, err := n.headVersion(ctx, bkt, addedVersions[len(addedVersions)-1])
if err != nil {
return &errors.DeleteError{Err: err, Object: obj.String()}
}
p := &CopyObjectParams{
SrcObject: sourceCopyVersion,
DstBucket: bkt.Name,
DstObject: obj.Name,
SrcSize: sourceCopyVersion.Size,
Header: map[string]string{versionsDelAttr: obj.VersionID},
}
if _, err := n.CopyObject(ctx, p); err != nil {
return err
}
} else {
p := &PutObjectParams{
Object: obj.Name,
Reader: bytes.NewReader(nil),
Header: map[string]string{
versionsDelAttr: obj.VersionID,
versionsDeleteMarkAttr: strconv.FormatBool(true),
},
}
if _, err := n.objectPut(ctx, bkt, p); err != nil {
return &errors.DeleteError{Err: err, Object: obj.String()}
}
}
} else {
p := &CopyObjectParams{
SrcObject: lastObject,
DstBucket: bkt.Name,
DstObject: obj.Name,
SrcSize: lastObject.Size,
Header: map[string]string{versionsDelAttr: obj.VersionID},
}
if _, err := n.CopyObject(ctx, p); err != nil {
return err
}
}
} else { } else {
p := &PutObjectParams{ p.Header[versionsDeleteMarkAttr] = "*"
Object: obj.Name, }
Reader: bytes.NewReader(nil), if _, err = n.objectPut(ctx, bkt, p); err != nil {
Header: map[string]string{versionsDeleteMarkAttr: strconv.FormatBool(true)}, return &errors.DeleteError{Err: err, Object: obj.String()}
}
if _, err := n.objectPut(ctx, bkt, p); err != nil {
return &errors.DeleteError{Err: err, Object: obj.String()}
}
} }
} else { } else {
ids, err = n.objectSearch(ctx, &findParams{cid: bkt.CID, val: obj.Name}) ids, err = n.objectSearch(ctx, &findParams{cid: bkt.CID, val: obj.Name})
@ -512,6 +511,23 @@ func (n *layer) deleteObject(ctx context.Context, bkt *BucketInfo, obj *Versione
return nil return nil
} }
func (n *layer) checkVersionsExists(ctx context.Context, bkt *BucketInfo, obj *VersionedObject) (*object.ID, error) {
id := object.NewID()
if err := id.Parse(obj.VersionID); err != nil {
return nil, &errors.DeleteError{Err: errors.GetAPIError(errors.ErrInvalidVersion), Object: obj.String()}
}
versions, err := n.headVersions(ctx, bkt, obj.Name)
if err != nil {
return nil, &errors.DeleteError{Err: err, Object: obj.String()}
}
if !contains(getExistedVersions(versions), obj.VersionID) {
return nil, &errors.DeleteError{Err: errors.GetAPIError(errors.ErrInvalidVersion), Object: obj.String()}
}
return id, nil
}
// DeleteObjects from the storage. // DeleteObjects from the storage.
func (n *layer) DeleteObjects(ctx context.Context, bucket string, objects []*VersionedObject) []error { func (n *layer) DeleteObjects(ctx context.Context, bucket string, objects []*VersionedObject) []error {
var errs = make([]error, 0, len(objects)) var errs = make([]error, 0, len(objects))
@ -560,24 +576,17 @@ func (n *layer) DeleteBucket(ctx context.Context, p *DeleteBucketParams) error {
} }
func (n *layer) ListObjectVersions(ctx context.Context, p *ListObjectVersionsParams) (*ListObjectVersionsInfo, error) { func (n *layer) ListObjectVersions(ctx context.Context, p *ListObjectVersionsParams) (*ListObjectVersionsInfo, error) {
var ( res := ListObjectVersionsInfo{}
res = ListObjectVersionsInfo{} versions := make(map[string]*objectVersions)
err error
bkt *BucketInfo
ids []*object.ID
latest = make(map[string]*ObjectVersionInfo)
)
if bkt, err = n.GetBucketInfo(ctx, p.Bucket); err != nil { bkt, err := n.GetBucketInfo(ctx, p.Bucket)
return nil, err if err != nil {
} else if ids, err = n.objectSearch(ctx, &findParams{cid: bkt.CID}); err != nil { return nil, err
}
ids, err := n.objectSearch(ctx, &findParams{cid: bkt.CID})
if err != nil {
return nil, err return nil, err
} }
versions := make([]*ObjectVersionInfo, 0, len(ids))
deleted := make([]*DeletedObjectInfo, 0, len(ids))
deletedVersions := []string{}
for _, id := range ids { for _, id := range ids {
meta, err := n.objectHead(ctx, bkt.CID, id) meta, err := n.objectHead(ctx, bkt.CID, id)
@ -585,102 +594,80 @@ func (n *layer) ListObjectVersions(ctx context.Context, p *ListObjectVersionsPar
n.log.Warn("could not fetch object meta", zap.Error(err)) n.log.Warn("could not fetch object meta", zap.Error(err))
continue continue
} }
if ov := objectVersionInfoFromMeta(bkt, meta, p.Prefix, p.Delimiter); ov != nil { if oi := objectInfoFromMeta(bkt, meta, p.Prefix, p.Delimiter); oi != nil {
if ov.Object.Name <= p.KeyMarker { if oi.Name <= p.KeyMarker {
continue continue
} }
if currentLatest, ok := latest[ov.Object.Name]; ok { if isSystem(oi) {
if less(currentLatest, ov) { continue
latest[ov.Object.Name] = ov
}
} else {
latest[ov.Object.Name] = ov
} }
if del := ov.Object.Headers[versionsDelAttr]; len(del) != 0 { if objVersions, ok := versions[oi.Name]; ok {
deletedVersions = append(deletedVersions, strings.Split(del, ",")...) objVersions.appendVersion(oi)
} versions[oi.Name] = objVersions
if parsed, err := strconv.ParseBool(ov.Object.Headers[versionsDeleteMarkAttr]); err == nil && parsed {
deleted = append(deleted, &DeletedObjectInfo{
Owner: ov.Object.Owner,
Key: ov.Object.Name,
VersionID: ov.VersionID,
LastModified: ov.Object.Created.Format(time.RFC3339),
})
} else { } else {
versions = append(versions, ov) objVersion := &objectVersions{}
objVersion.appendVersion(oi)
versions[oi.Name] = objVersion
} }
} }
} }
sort.Slice(versions, func(i, j int) bool { for _, v := range versions {
if contains(deletedVersions, versions[i].VersionID) { existed, deleted := triageVersions(v)
return true res.Version = append(res.Version, existed...)
} res.DeleteMarker = append(res.DeleteMarker, deleted...)
if contains(deletedVersions, versions[j].VersionID) { }
return false
} sort.Slice(res.Version, func(i, j int) bool {
if versions[i].Object.Name == versions[j].Object.Name { return res.Version[i].Object.Name < res.Version[j].Object.Name
if versions[i].CreationEpoch == versions[j].CreationEpoch { })
return versions[i].VersionID < versions[j].VersionID sort.Slice(res.DeleteMarker, func(i, j int) bool {
} return res.DeleteMarker[i].Object.Name < res.DeleteMarker[j].Object.Name
return versions[i].CreationEpoch < versions[j].CreationEpoch
}
return versions[i].Object.Name < versions[j].Object.Name
}) })
for i, objVersion := range versions {
if i == len(versions)-1 || objVersion.Object.Name != versions[i+1].Object.Name {
objVersion.IsLatest = true
}
}
for i, objVersion := range versions {
if !contains(deletedVersions, objVersion.VersionID) {
versions = versions[i:]
break
}
}
for i, objVersion := range deleted {
if !contains(deletedVersions, objVersion.VersionID) {
deleted = deleted[i:]
break
}
}
//if len(versions) > p.MaxKeys {
// res.IsTruncated = true
//
// lastVersion := versions[p.MaxKeys-1]
// res.KeyMarker = lastVersion.Object.Name
// res.VersionIDMarker = lastVersion.VersionID
//
// nextVersion := versions[p.MaxKeys]
// res.NextKeyMarker = nextVersion.Object.Name
// res.NextVersionIDMarker = nextVersion.VersionID
//
// versions = versions[:p.MaxKeys]
//}
//
//for _, ov := range versions {
// if isDir := uniqNames[ov.Object.Name]; isDir {
// res.CommonPrefixes = append(res.CommonPrefixes, &ov.Object.Name)
// } else {
// res.Version = append(res.Version, ov)
// }
//}
res.Version = versions
res.DeleteMarker = deleted
return &res, nil return &res, nil
} }
func less(ov1, ov2 *ObjectVersionInfo) bool { func sortVersions(versions []*ObjectInfo) {
sort.Slice(versions, func(i, j int) bool {
return less(versions[i], versions[j])
})
}
func triageVersions(objectVersions *objectVersions) ([]*ObjectVersionInfo, []*ObjectVersionInfo) {
if objectVersions == nil || len(objectVersions.objects) == 0 {
return nil, nil
}
sortVersions(objectVersions.objects)
var resVersion []*ObjectVersionInfo
var resDelMarkVersions []*ObjectVersionInfo
isLatest := true
for i := len(objectVersions.objects) - 1; i >= 0; i-- {
version := objectVersions.objects[i]
if contains(objectVersions.delList, version.Version()) {
continue
}
ovi := &ObjectVersionInfo{Object: version, IsLatest: isLatest}
isLatest = false
if len(version.Headers[versionsDeleteMarkAttr]) == 0 {
resVersion = append(resVersion, ovi)
} else {
resDelMarkVersions = append(resDelMarkVersions, ovi)
}
}
return resVersion, resDelMarkVersions
}
func less(ov1, ov2 *ObjectInfo) bool {
if ov1.CreationEpoch == ov2.CreationEpoch { if ov1.CreationEpoch == ov2.CreationEpoch {
return ov1.VersionID < ov2.VersionID return ov1.Version() < ov2.Version()
} }
return ov1.CreationEpoch < ov2.CreationEpoch return ov1.CreationEpoch < ov2.CreationEpoch
} }
@ -711,7 +698,7 @@ func (n *layer) PutBucketVersioning(ctx context.Context, p *PutVersioningParams)
attributes := make([]*object.Attribute, 0, 3) attributes := make([]*object.Attribute, 0, 3)
filename := object.NewAttribute() filename := object.NewAttribute()
filename.SetKey(object.AttributeFileName) filename.SetKey(objectSystemAttributeName)
filename.SetValue(bktVersionSettingsObject) filename.SetValue(bktVersionSettingsObject)
createdAt := object.NewAttribute() createdAt := object.NewAttribute()
@ -719,11 +706,11 @@ func (n *layer) PutBucketVersioning(ctx context.Context, p *PutVersioningParams)
createdAt.SetValue(strconv.FormatInt(time.Now().UTC().Unix(), 10)) createdAt.SetValue(strconv.FormatInt(time.Now().UTC().Unix(), 10))
versioningIgnore := object.NewAttribute() versioningIgnore := object.NewAttribute()
versioningIgnore.SetKey("S3-Versions-ignore") versioningIgnore.SetKey(attrVersionsIgnore)
versioningIgnore.SetValue(strconv.FormatBool(true)) versioningIgnore.SetValue(strconv.FormatBool(true))
settingsVersioningEnabled := object.NewAttribute() settingsVersioningEnabled := object.NewAttribute()
settingsVersioningEnabled.SetKey("S3-Settings-Versioning-enabled") settingsVersioningEnabled.SetKey(attrSettingsVersioningEnabled)
settingsVersioningEnabled.SetValue(strconv.FormatBool(p.Settings.VersioningEnabled)) settingsVersioningEnabled.SetValue(strconv.FormatBool(p.Settings.VersioningEnabled))
attributes = append(attributes, filename, createdAt, versioningIgnore, settingsVersioningEnabled) attributes = append(attributes, filename, createdAt, versioningIgnore, settingsVersioningEnabled)

248
api/layer/object.go
View file

@ -3,7 +3,6 @@ package layer
import ( import (
"context" "context"
"errors" "errors"
"fmt"
"io" "io"
"net/url" "net/url"
"sort" "sort"
@ -14,14 +13,16 @@ import (
"github.com/nspcc-dev/neofs-api-go/pkg/client" "github.com/nspcc-dev/neofs-api-go/pkg/client"
cid "github.com/nspcc-dev/neofs-api-go/pkg/container/id" cid "github.com/nspcc-dev/neofs-api-go/pkg/container/id"
"github.com/nspcc-dev/neofs-api-go/pkg/object" "github.com/nspcc-dev/neofs-api-go/pkg/object"
"github.com/nspcc-dev/neofs-api-go/pkg/owner"
apiErrors "github.com/nspcc-dev/neofs-s3-gw/api/errors" apiErrors "github.com/nspcc-dev/neofs-s3-gw/api/errors"
"go.uber.org/zap" "go.uber.org/zap"
) )
type ( type (
findParams struct { findParams struct {
val string attr string
cid *cid.ID val string
cid *cid.ID
} }
getParams struct { getParams struct {
@ -78,7 +79,11 @@ func (n *layer) objectSearch(ctx context.Context, p *findParams) ([]*object.ID,
if filename, err := url.QueryUnescape(p.val); err != nil { if filename, err := url.QueryUnescape(p.val); err != nil {
return nil, err return nil, err
} else if filename != "" { } else if filename != "" {
opts.AddFilter(object.AttributeFileName, filename, object.MatchStringEqual) if p.attr == "" {
opts.AddFilter(object.AttributeFileName, filename, object.MatchStringEqual)
} else {
opts.AddFilter(p.attr, filename, object.MatchStringEqual)
}
} }
return n.pool.SearchObject(ctx, new(client.SearchObjectParams).WithContainerID(p.cid).WithSearchFilters(opts), n.BearerOpt(ctx)) return n.pool.SearchObject(ctx, new(client.SearchObjectParams).WithContainerID(p.cid).WithSearchFilters(opts), n.BearerOpt(ctx))
} }
@ -123,60 +128,59 @@ func (n *layer) objectRange(ctx context.Context, p *getParams) ([]byte, error) {
// objectPut into NeoFS, took payload from io.Reader. // objectPut into NeoFS, took payload from io.Reader.
func (n *layer) objectPut(ctx context.Context, bkt *BucketInfo, p *PutObjectParams) (*ObjectInfo, error) { func (n *layer) objectPut(ctx context.Context, bkt *BucketInfo, p *PutObjectParams) (*ObjectInfo, error) {
var ( own := n.Owner(ctx)
err error obj, err := url.QueryUnescape(p.Object)
obj string if err != nil {
own = n.Owner(ctx)
)
if p.Object == bktVersionSettingsObject {
return nil, fmt.Errorf("trying put bucket settings object")
}
if obj, err = url.QueryUnescape(p.Object); err != nil {
return nil, err return nil, err
} }
versioningEnabled := n.isVersioningEnabled(ctx, bkt) versioningEnabled := n.isVersioningEnabled(ctx, bkt)
lastVersionInfo, err := n.headLastVersion(ctx, bkt, p.Object) versions, err := n.headVersions(ctx, bkt, obj)
if err != nil && !apiErrors.IsS3Error(err, apiErrors.ErrNoSuchKey) { if err != nil && !apiErrors.IsS3Error(err, apiErrors.ErrNoSuchKey) {
return nil, err return nil, err
} }
idsToDeleteArr := updateCRDT2PSetHeaders(p, versions, versioningEnabled)
attributes := make([]*object.Attribute, 0, len(p.Header)+1) rawObject := formRawObject(p, bkt.CID, own, obj)
var idsToDeleteArr []*object.ID r := newDetector(p.Reader)
if lastVersionInfo != nil {
if versioningEnabled {
versionsAddedStr := lastVersionInfo.Headers[versionsAddAttr]
if len(versionsAddedStr) != 0 {
versionsAddedStr += ","
}
versionsAddedStr += lastVersionInfo.ID().String()
p.Header[versionsAddAttr] = versionsAddedStr
deleted := p.Header[versionsDelAttr] ops := new(client.PutObjectParams).WithObject(rawObject.Object()).WithPayloadReader(r)
if delVersions := lastVersionInfo.Headers[versionsDelAttr]; len(delVersions) != 0 { oid, err := n.pool.PutObject(ctx, ops, n.BearerOpt(ctx))
if len(deleted) == 0 { if err != nil {
deleted = delVersions return nil, err
} else { }
deleted = delVersions + "," + deleted
}
}
if len(deleted) != 0 {
p.Header[versionsDelAttr] = deleted
}
} else {
versionsDeletedStr := lastVersionInfo.Headers[versionsDelAttr]
if len(versionsDeletedStr) != 0 {
versionsDeletedStr += ","
}
versionsDeletedStr += lastVersionInfo.ID().String()
p.Header[versionsDelAttr] = versionsDeletedStr
idsToDeleteArr = append(idsToDeleteArr, lastVersionInfo.ID()) meta, err := n.objectHead(ctx, bkt.CID, oid)
if err != nil {
return nil, err
}
for _, id := range idsToDeleteArr {
if err = n.objectDelete(ctx, bkt.CID, id); err != nil {
n.log.Warn("couldn't delete object",
zap.Stringer("version id", id),
zap.Error(err))
} }
} }
return &ObjectInfo{
id: oid,
bucketID: bkt.CID,
Owner: own,
Bucket: p.Bucket,
Name: p.Object,
Size: p.Size,
Created: time.Now(),
CreationEpoch: meta.CreationEpoch(),
Headers: p.Header,
ContentType: r.contentType,
HashSum: meta.PayloadChecksum().String(),
}, nil
}
func formRawObject(p *PutObjectParams, bktID *cid.ID, own *owner.ID, obj string) *object.RawObject {
attributes := make([]*object.Attribute, 0, len(p.Header)+2)
filename := object.NewAttribute() filename := object.NewAttribute()
filename.SetKey(object.AttributeFileName) filename.SetKey(object.AttributeFileName)
filename.SetValue(obj) filename.SetValue(obj)
@ -197,55 +201,68 @@ func (n *layer) objectPut(ctx context.Context, bkt *BucketInfo, p *PutObjectPara
raw := object.NewRaw() raw := object.NewRaw()
raw.SetOwnerID(own) raw.SetOwnerID(own)
raw.SetContainerID(bkt.CID) raw.SetContainerID(bktID)
raw.SetAttributes(attributes...) raw.SetAttributes(attributes...)
r := newDetector(p.Reader) return raw
}
ops := new(client.PutObjectParams).WithObject(raw.Object()).WithPayloadReader(r) func updateCRDT2PSetHeaders(p *PutObjectParams, versions *objectVersions, versioningEnabled bool) []*object.ID {
oid, err := n.pool.PutObject( var idsToDeleteArr []*object.ID
ctx, if versions == nil {
ops, return idsToDeleteArr
n.BearerOpt(ctx),
)
if err != nil {
return nil, err
} }
meta, err := n.objectHead(ctx, bkt.CID, oid) if versioningEnabled {
if err != nil { if len(versions.addList) != 0 {
return nil, err p.Header[versionsAddAttr] = versions.getAddHeader()
} }
if err = n.objCache.Put(addr, *meta); err != nil { deleted := versions.getDelHeader()
n.log.Error("couldn't cache an object", zap.Error(err)) // p.Header[versionsDelAttr] can be not empty when deleting specific version
} if delAttr := p.Header[versionsDelAttr]; len(delAttr) != 0 {
if len(deleted) != 0 {
p.Header[versionsDelAttr] = deleted + "," + delAttr
} else {
p.Header[versionsDelAttr] = delAttr
}
} else if len(deleted) != 0 {
p.Header[versionsDelAttr] = deleted
}
} else {
versionsDeletedStr := versions.getDelHeader()
if len(versionsDeletedStr) != 0 {
versionsDeletedStr += ","
}
objInfo := &ObjectInfo{ lastVersion := versions.getLast()
id: oid, p.Header[versionsDelAttr] = versionsDeletedStr + lastVersion.Version()
idsToDeleteArr = append(idsToDeleteArr, lastVersion.ID())
Owner: own, for _, version := range versions.objects {
Bucket: p.Bucket, if contains(versions.delList, version.Version()) {
Name: p.Object, idsToDeleteArr = append(idsToDeleteArr, version.ID())
Size: p.Size, }
Created: time.Now(),
Headers: p.Header,
ContentType: r.contentType,
HashSum: meta.PayloadChecksum().String(),
}
for _, id := range idsToDeleteArr {
if err = n.objectDelete(ctx, bkt.CID, id); err != nil {
n.log.Warn("couldn't delete object",
zap.Stringer("version id", id),
zap.Error(err))
} }
} }
return objInfo, nil return idsToDeleteArr
} }
func (n *layer) headLastVersion(ctx context.Context, bkt *BucketInfo, objectName string) (*ObjectInfo, error) { func (n *layer) headLastVersionIfNotDeleted(ctx context.Context, bkt *BucketInfo, objectName string) (*ObjectInfo, error) {
versions, err := n.headVersions(ctx, bkt, objectName)
if err != nil {
return nil, err
}
lastVersion := versions.getLast()
if lastVersion == nil {
return nil, apiErrors.GetAPIError(apiErrors.ErrNoSuchKey)
}
return lastVersion, nil
}
func (n *layer) headVersions(ctx context.Context, bkt *BucketInfo, objectName string) (*objectVersions, error) {
ids, err := n.objectSearch(ctx, &findParams{cid: bkt.CID, val: objectName}) ids, err := n.objectSearch(ctx, &findParams{cid: bkt.CID, val: objectName})
if err != nil { if err != nil {
return nil, err return nil, err
@ -255,7 +272,7 @@ func (n *layer) headLastVersion(ctx context.Context, bkt *BucketInfo, objectName
return nil, apiErrors.GetAPIError(apiErrors.ErrNoSuchKey) return nil, apiErrors.GetAPIError(apiErrors.ErrNoSuchKey)
} }
infos := make([]*object.Object, 0, len(ids)) versions := &objectVersions{}
for _, id := range ids { for _, id := range ids {
meta, err := n.objectHead(ctx, bkt.CID, id) meta, err := n.objectHead(ctx, bkt.CID, id)
if err != nil { if err != nil {
@ -265,14 +282,15 @@ func (n *layer) headLastVersion(ctx context.Context, bkt *BucketInfo, objectName
zap.Error(err)) zap.Error(err))
continue continue
} }
infos = append(infos, meta) if oi := objectInfoFromMeta(bkt, meta, "", ""); oi != nil {
if isSystem(oi) {
continue
}
versions.appendVersion(oi)
}
} }
sort.Slice(infos, func(i, j int) bool { return versions, nil
return infos[i].CreationEpoch() < infos[j].CreationEpoch() || (infos[i].CreationEpoch() == infos[j].CreationEpoch() && infos[i].ID().String() < infos[j].ID().String())
})
return objectInfoFromMeta(bkt, infos[len(infos)-1], "", ""), nil
} }
func (n *layer) headVersion(ctx context.Context, bkt *BucketInfo, versionID string) (*ObjectInfo, error) { func (n *layer) headVersion(ctx context.Context, bkt *BucketInfo, versionID string) (*ObjectInfo, error) {
@ -378,17 +396,12 @@ func (n *layer) ListObjectsV2(ctx context.Context, p *ListObjectsParamsV2) (*Lis
} }
func (n *layer) listSortedObjectsFromNeoFS(ctx context.Context, p allObjectParams) ([]*ObjectInfo, error) { func (n *layer) listSortedObjectsFromNeoFS(ctx context.Context, p allObjectParams) ([]*ObjectInfo, error) {
var ( ids, err := n.objectSearch(ctx, &findParams{cid: p.Bucket.CID})
err error if err != nil {
ids []*object.ID
uniqNames = make(map[string]bool)
)
if ids, err = n.objectSearch(ctx, &findParams{cid: p.Bucket.CID}); err != nil {
return nil, err return nil, err
} }
objects := make([]*ObjectInfo, 0, len(ids)) versions := make(map[string]*objectVersions, len(ids)/2)
for _, id := range ids { for _, id := range ids {
meta, err := n.objectHead(ctx, p.Bucket.CID, id) meta, err := n.objectHead(ctx, p.Bucket.CID, id)
@ -397,14 +410,26 @@ func (n *layer) listSortedObjectsFromNeoFS(ctx context.Context, p allObjectParam
continue continue
} }
if oi := objectInfoFromMeta(p.Bucket, meta, p.Prefix, p.Delimiter); oi != nil { if oi := objectInfoFromMeta(p.Bucket, meta, p.Prefix, p.Delimiter); oi != nil {
// use only unique dir names if isSystem(oi) {
if _, ok := uniqNames[oi.Name]; ok {
continue continue
} }
uniqNames[oi.Name] = oi.isDir if objVersions, ok := versions[oi.Name]; ok {
objVersions.appendVersion(oi)
versions[oi.Name] = objVersions
} else {
objVersion := &objectVersions{}
objVersion.appendVersion(oi)
versions[oi.Name] = objVersion
}
}
}
objects = append(objects, oi) objects := make([]*ObjectInfo, 0, len(versions))
for _, v := range versions {
lastVersion := v.getLast()
if lastVersion != nil {
objects = append(objects, lastVersion)
} }
} }
@ -415,6 +440,29 @@ func (n *layer) listSortedObjectsFromNeoFS(ctx context.Context, p allObjectParam
return objects, nil return objects, nil
} }
func getExistedVersions(versions *objectVersions) []string {
var res []string
for _, add := range versions.addList {
if !contains(versions.delList, add) {
res = append(res, add)
}
}
return res
}
func splitVersions(header string) []string {
if len(header) == 0 {
return nil
}
return strings.Split(header, ",")
}
func isSystem(obj *ObjectInfo) bool {
return len(obj.Headers[objectSystemAttributeName]) > 0 ||
len(obj.Headers[attrVersionsIgnore]) > 0
}
func trimAfterObjectName(startAfter string, objects []*ObjectInfo) []*ObjectInfo { func trimAfterObjectName(startAfter string, objects []*ObjectInfo) []*ObjectInfo {
if len(objects) != 0 && objects[len(objects)-1].Name <= startAfter { if len(objects) != 0 && objects[len(objects)-1].Name <= startAfter {
return nil return nil

75
api/layer/util.go
View file

@ -9,7 +9,6 @@ import (
"time" "time"
cid "github.com/nspcc-dev/neofs-api-go/pkg/container/id" cid "github.com/nspcc-dev/neofs-api-go/pkg/container/id"
"github.com/nspcc-dev/neofs-api-go/pkg/object" "github.com/nspcc-dev/neofs-api-go/pkg/object"
"github.com/nspcc-dev/neofs-api-go/pkg/owner" "github.com/nspcc-dev/neofs-api-go/pkg/owner"
"github.com/nspcc-dev/neofs-s3-gw/api" "github.com/nspcc-dev/neofs-s3-gw/api"
@ -19,18 +18,19 @@ import (
type ( type (
// ObjectInfo holds S3 object data. // ObjectInfo holds S3 object data.
ObjectInfo struct { ObjectInfo struct {
id *object.ID id *object.ID
isDir bool bucketID *cid.ID
isDir bool
Bucket string Bucket string
bucketID *cid.ID Name string
Name string Size int64
Size int64 ContentType string
ContentType string Created time.Time
Created time.Time CreationEpoch uint64
HashSum string HashSum string
Owner *owner.ID Owner *owner.ID
Headers map[string]string Headers map[string]string
} }
// ListObjectsInfo contains common fields of data for ListObjectsV1 and ListObjectsV2. // ListObjectsInfo contains common fields of data for ListObjectsV1 and ListObjectsV2.
@ -54,19 +54,8 @@ type (
// ObjectVersionInfo stores info about objects versions. // ObjectVersionInfo stores info about objects versions.
ObjectVersionInfo struct { ObjectVersionInfo struct {
Object *ObjectInfo Object *ObjectInfo
IsLatest bool IsLatest bool
VersionID string
CreationEpoch uint64
}
// DeletedObjectInfo stores info about deleted versions of objects.
DeletedObjectInfo struct {
Owner *owner.ID
Key string
VersionID string
IsLatest bool
LastModified string
} }
// ListObjectVersionsInfo stores info and list of objects' versions. // ListObjectVersionsInfo stores info and list of objects' versions.
@ -77,7 +66,7 @@ type (
NextKeyMarker string NextKeyMarker string
NextVersionIDMarker string NextVersionIDMarker string
Version []*ObjectVersionInfo Version []*ObjectVersionInfo
DeleteMarker []*DeletedObjectInfo DeleteMarker []*ObjectVersionInfo
VersionIDMarker string VersionIDMarker string
} }
) )
@ -137,29 +126,22 @@ func objectInfoFromMeta(bkt *BucketInfo, meta *object.Object, prefix, delimiter
} }
return &ObjectInfo{ return &ObjectInfo{
id: meta.ID(), id: meta.ID(),
isDir: isDir, bucketID: bkt.CID,
isDir: isDir,
Bucket: bkt.Name, Bucket: bkt.Name,
bucketID: bkt.CID, Name: filename,
Name: filename, Created: creation,
Created: creation, CreationEpoch: meta.CreationEpoch(),
ContentType: mimeType, ContentType: mimeType,
Headers: userHeaders, Headers: userHeaders,
Owner: meta.OwnerID(), Owner: meta.OwnerID(),
Size: size, Size: size,
HashSum: meta.PayloadChecksum().String(), HashSum: meta.PayloadChecksum().String(),
} }
} }
func objectVersionInfoFromMeta(bkt *BucketInfo, meta *object.Object, prefix, delimiter string) *ObjectVersionInfo {
oi := objectInfoFromMeta(bkt, meta, prefix, delimiter)
if oi == nil {
return nil
}
return &ObjectVersionInfo{Object: oi, VersionID: meta.ID().String(), CreationEpoch: meta.CreationEpoch()}
}
func filenameFromObject(o *object.Object) string { func filenameFromObject(o *object.Object) string {
var name = o.ID().String() var name = o.ID().String()
for _, attr := range o.Attributes() { for _, attr := range o.Attributes() {
@ -179,6 +161,9 @@ func NameFromString(name string) (string, string) {
// ID returns object ID from ObjectInfo. // ID returns object ID from ObjectInfo.
func (o *ObjectInfo) ID() *object.ID { return o.id } func (o *ObjectInfo) ID() *object.ID { return o.id }
// Version returns object version from ObjectInfo.
func (o *ObjectInfo) Version() string { return o.id.String() }
// CID returns bucket ID from ObjectInfo. // CID returns bucket ID from ObjectInfo.
func (o *ObjectInfo) CID() *cid.ID { return o.bucketID } func (o *ObjectInfo) CID() *cid.ID { return o.bucketID }

608
api/layer/versioning_test.go Normal file
View file

@ -0,0 +1,608 @@
package layer
import (
"bytes"
"context"
"crypto/rand"
"crypto/sha256"
"fmt"
"io"
"strings"
"testing"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
"github.com/nspcc-dev/neofs-api-go/pkg/acl/eacl"
"github.com/nspcc-dev/neofs-api-go/pkg/client"
"github.com/nspcc-dev/neofs-api-go/pkg/container"
cid "github.com/nspcc-dev/neofs-api-go/pkg/container/id"
"github.com/nspcc-dev/neofs-api-go/pkg/object"
"github.com/nspcc-dev/neofs-api-go/pkg/owner"
"github.com/nspcc-dev/neofs-api-go/pkg/session"
"github.com/nspcc-dev/neofs-api-go/pkg/token"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
"github.com/nspcc-dev/neofs-sdk-go/pkg/logger"
"github.com/nspcc-dev/neofs-sdk-go/pkg/pool"
"github.com/stretchr/testify/require"
)
type testPool struct {
objects map[string]*object.Object
containers map[string]*container.Container
currentEpoch uint64
}
func newTestPool() *testPool {
return &testPool{
objects: make(map[string]*object.Object),
containers: make(map[string]*container.Container),
}
}
func (t *testPool) PutObject(ctx context.Context, params *client.PutObjectParams, option ...client.CallOption) (*object.ID, error) {
b := make([]byte, 32)
if _, err := io.ReadFull(rand.Reader, b); err != nil {
return nil, err
}
oid := object.NewID()
oid.SetSHA256(sha256.Sum256(b))
raw := object.NewRawFrom(params.Object())
raw.SetID(oid)
raw.SetCreationEpoch(t.currentEpoch)
t.currentEpoch++
if params.PayloadReader() != nil {
all, err := io.ReadAll(params.PayloadReader())
if err != nil {
return nil, err
}
raw.SetPayload(all)
}
addr := object.NewAddress()
addr.SetObjectID(raw.ID())
addr.SetContainerID(raw.ContainerID())
t.objects[addr.String()] = raw.Object()
return raw.ID(), nil
}
func (t *testPool) DeleteObject(ctx context.Context, params *client.DeleteObjectParams, option ...client.CallOption) error {
delete(t.objects, params.Address().String())
return nil
}
func (t *testPool) GetObject(ctx context.Context, params *client.GetObjectParams, option ...client.CallOption) (*object.Object, error) {
if obj, ok := t.objects[params.Address().String()]; ok {
if params.PayloadWriter() != nil {
_, err := params.PayloadWriter().Write(obj.Payload())
if err != nil {
return nil, err
}
}
return obj, nil
}
return nil, fmt.Errorf("object not found " + params.Address().String())
}
func (t *testPool) GetObjectHeader(ctx context.Context, params *client.ObjectHeaderParams, option ...client.CallOption) (*object.Object, error) {
p := new(client.GetObjectParams).WithAddress(params.Address())
return t.GetObject(ctx, p)
}
func (t *testPool) ObjectPayloadRangeData(ctx context.Context, params *client.RangeDataParams, option ...client.CallOption) ([]byte, error) {
panic("implement me")
}
func (t *testPool) ObjectPayloadRangeSHA256(ctx context.Context, params *client.RangeChecksumParams, option ...client.CallOption) ([][32]byte, error) {
panic("implement me")
}
func (t *testPool) ObjectPayloadRangeTZ(ctx context.Context, params *client.RangeChecksumParams, option ...client.CallOption) ([][64]byte, error) {
panic("implement me")
}
func (t *testPool) SearchObject(ctx context.Context, params *client.SearchObjectParams, option ...client.CallOption) ([]*object.ID, error) {
cidStr := params.ContainerID().String()
var res []*object.ID
if len(params.SearchFilters()) == 1 {
for k, v := range t.objects {
if strings.Contains(k, cidStr) {
res = append(res, v.ID())
}
}
return res, nil
}
filter := params.SearchFilters()[1]
if len(params.SearchFilters()) != 2 || filter.Operation() != object.MatchStringEqual ||
(filter.Header() != object.AttributeFileName && filter.Header() != objectSystemAttributeName) {
return nil, fmt.Errorf("usupported filters")
}
for k, v := range t.objects {
if strings.Contains(k, cidStr) && isMatched(v.Attributes(), filter) {
res = append(res, v.ID())
}
}
return res, nil
}
func isMatched(attributes []*object.Attribute, filter object.SearchFilter) bool {
for _, attr := range attributes {
if attr.Key() == filter.Header() && attr.Value() == filter.Value() {
return true
}
}
return false
}
func (t *testPool) PutContainer(ctx context.Context, container *container.Container, option ...client.CallOption) (*cid.ID, error) {
b := make([]byte, 32)
if _, err := io.ReadFull(rand.Reader, b); err != nil {
return nil, err
}
id := cid.New()
id.SetSHA256(sha256.Sum256(b))
t.containers[id.String()] = container
return id, nil
}
func (t *testPool) GetContainer(ctx context.Context, id *cid.ID, option ...client.CallOption) (*container.Container, error) {
for k, v := range t.containers {
if k == id.String() {
return v, nil
}
}
return nil, fmt.Errorf("container not found " + id.String())
}
func (t *testPool) ListContainers(ctx context.Context, id *owner.ID, option ...client.CallOption) ([]*cid.ID, error) {
var res []*cid.ID
for k := range t.containers {
cID := cid.New()
if err := cID.Parse(k); err != nil {
return nil, err
}
res = append(res, cID)
}
return res, nil
}
func (t *testPool) DeleteContainer(ctx context.Context, id *cid.ID, option ...client.CallOption) error {
delete(t.containers, id.String())
return nil
}
func (t *testPool) GetEACL(ctx context.Context, id *cid.ID, option ...client.CallOption) (*client.EACLWithSignature, error) {
panic("implement me")
}
func (t *testPool) SetEACL(ctx context.Context, table *eacl.Table, option ...client.CallOption) error {
panic("implement me")
}
func (t *testPool) AnnounceContainerUsedSpace(ctx context.Context, announcements []container.UsedSpaceAnnouncement, option ...client.CallOption) error {
panic("implement me")
}
func (t *testPool) Connection() (client.Client, *session.Token, error) {
panic("implement me")
}
func (t *testPool) OwnerID() *owner.ID {
return nil
}
func (t *testPool) WaitForContainerPresence(ctx context.Context, id *cid.ID, params *pool.ContainerPollingParams) error {
return nil
}
func (tc *testContext) putObject(content []byte) *ObjectInfo {
objInfo, err := tc.layer.PutObject(tc.ctx, &PutObjectParams{
Bucket: tc.bkt,
Object: tc.obj,
Size: int64(len(content)),
Reader: bytes.NewReader(content),
Header: make(map[string]string),
})
require.NoError(tc.t, err)
return objInfo
}
func (tc *testContext) getObject(objectName, versionID string, needError bool) (*ObjectInfo, []byte) {
objInfo, err := tc.layer.GetObjectInfo(tc.ctx, &HeadObjectParams{
Bucket: tc.bkt,
Object: objectName,
VersionID: versionID,
})
if needError {
require.Error(tc.t, err)
return nil, nil
}
require.NoError(tc.t, err)
content := bytes.NewBuffer(nil)
err = tc.layer.GetObject(tc.ctx, &GetObjectParams{
ObjectInfo: objInfo,
Writer: content,
VersionID: versionID,
})
require.NoError(tc.t, err)
return objInfo, content.Bytes()
}
func (tc *testContext) deleteObject(objectName, versionID string) {
errs := tc.layer.DeleteObjects(tc.ctx, tc.bkt, []*VersionedObject{
{Name: objectName, VersionID: versionID},
})
for _, err := range errs {
require.NoError(tc.t, err)
}
}
func (tc *testContext) listObjectsV1() []*ObjectInfo {
res, err := tc.layer.ListObjectsV1(tc.ctx, &ListObjectsParamsV1{
ListObjectsParamsCommon: ListObjectsParamsCommon{
Bucket: tc.bkt,
MaxKeys: 1000,
},
})
require.NoError(tc.t, err)
return res.Objects
}
func (tc *testContext) listObjectsV2() []*ObjectInfo {
res, err := tc.layer.ListObjectsV2(tc.ctx, &ListObjectsParamsV2{
ListObjectsParamsCommon: ListObjectsParamsCommon{
Bucket: tc.bkt,
MaxKeys: 1000,
},
})
require.NoError(tc.t, err)
return res.Objects
}
func (tc *testContext) listVersions() *ListObjectVersionsInfo {
res, err := tc.layer.ListObjectVersions(tc.ctx, &ListObjectVersionsParams{
Bucket: tc.bkt,
MaxKeys: 1000,
})
require.NoError(tc.t, err)
return res
}
func (tc *testContext) checkListObjects(ids ...*object.ID) {
objs := tc.listObjectsV1()
require.Equal(tc.t, len(ids), len(objs))
for _, id := range ids {
require.Contains(tc.t, ids, id)
}
objs = tc.listObjectsV2()
require.Equal(tc.t, len(ids), len(objs))
for _, id := range ids {
require.Contains(tc.t, ids, id)
}
}
type testContext struct {
t *testing.T
ctx context.Context
layer Client
bkt string
bktID *cid.ID
obj string
testPool *testPool
}
func prepareContext(t *testing.T) *testContext {
key, err := keys.NewPrivateKey()
require.NoError(t, err)
ctx := context.WithValue(context.Background(), api.BoxData, &accessbox.Box{
Gate: &accessbox.GateData{
BearerToken: token.NewBearerToken(),
GateKey: key.PublicKey(),
},
})
l, err := logger.New(logger.WithTraceLevel("panic"))
require.NoError(t, err)
tp := newTestPool()
bktName := "testbucket1"
cnr := container.New(container.WithAttribute(container.AttributeName, bktName))
bktID, err := tp.PutContainer(ctx, cnr)
require.NoError(t, err)
return &testContext{
ctx: ctx,
layer: NewLayer(l, tp),
bkt: bktName,
bktID: bktID,
obj: "obj1",
t: t,
testPool: tp,
}
}
func TestSimpleVersioning(t *testing.T) {
tc := prepareContext(t)
_, err := tc.layer.PutBucketVersioning(tc.ctx, &PutVersioningParams{
Bucket: tc.bkt,
Settings: &BucketSettings{VersioningEnabled: true},
})
require.NoError(t, err)
obj1Content1 := []byte("content obj1 v1")
obj1v1 := tc.putObject(obj1Content1)
obj1Content2 := []byte("content obj1 v2")
obj1v2 := tc.putObject(obj1Content2)
objv2, buffer2 := tc.getObject(tc.obj, "", false)
require.Equal(t, obj1Content2, buffer2)
require.Contains(t, objv2.Headers[versionsAddAttr], obj1v1.ID().String())
_, buffer1 := tc.getObject(tc.obj, obj1v1.ID().String(), false)
require.Equal(t, obj1Content1, buffer1)
tc.checkListObjects(obj1v2.ID())
}
func TestSimpleNoVersioning(t *testing.T) {
tc := prepareContext(t)
obj1Content1 := []byte("content obj1 v1")
obj1v1 := tc.putObject(obj1Content1)
obj1Content2 := []byte("content obj1 v2")
obj1v2 := tc.putObject(obj1Content2)
objv2, buffer2 := tc.getObject(tc.obj, "", false)
require.Equal(t, obj1Content2, buffer2)
require.Contains(t, objv2.Headers[versionsDelAttr], obj1v1.ID().String())
tc.getObject(tc.obj, obj1v1.ID().String(), true)
tc.checkListObjects(obj1v2.ID())
}
func TestVersioningDeleteObject(t *testing.T) {
tc := prepareContext(t)
_, err := tc.layer.PutBucketVersioning(tc.ctx, &PutVersioningParams{
Bucket: tc.bkt,
Settings: &BucketSettings{VersioningEnabled: true},
})
require.NoError(t, err)
tc.putObject([]byte("content obj1 v1"))
tc.putObject([]byte("content obj1 v2"))
tc.deleteObject(tc.obj, "")
tc.getObject(tc.obj, "", true)
tc.checkListObjects()
}
func TestVersioningDeleteSpecificObjectVersion(t *testing.T) {
tc := prepareContext(t)
_, err := tc.layer.PutBucketVersioning(tc.ctx, &PutVersioningParams{
Bucket: tc.bkt,
Settings: &BucketSettings{VersioningEnabled: true},
})
require.NoError(t, err)
tc.putObject([]byte("content obj1 v1"))
objV2Info := tc.putObject([]byte("content obj1 v2"))
objV3Content := []byte("content obj1 v3")
objV3Info := tc.putObject(objV3Content)
tc.deleteObject(tc.obj, objV2Info.Version())
tc.getObject(tc.obj, objV2Info.Version(), true)
_, buffer3 := tc.getObject(tc.obj, "", false)
require.Equal(t, objV3Content, buffer3)
tc.deleteObject(tc.obj, "")
tc.getObject(tc.obj, "", true)
for _, ver := range tc.listVersions().DeleteMarker {
if ver.IsLatest {
tc.deleteObject(tc.obj, ver.Object.Version())
}
}
resInfo, buffer := tc.getObject(tc.obj, "", false)
require.Equal(t, objV3Content, buffer)
require.Equal(t, objV3Info.Version(), resInfo.Version())
}
func TestNoVersioningDeleteObject(t *testing.T) {
tc := prepareContext(t)
tc.putObject([]byte("content obj1 v1"))
tc.putObject([]byte("content obj1 v2"))
tc.deleteObject(tc.obj, "")
tc.getObject(tc.obj, "", true)
tc.checkListObjects()
}
func TestGetLastVersion(t *testing.T) {
obj1 := getTestObjectInfo(1, getOID(1), "", "", "")
obj1V2 := getTestObjectInfo(1, getOID(2), "", "", "")
obj2 := getTestObjectInfo(2, getOID(2), obj1.Version(), "", "")
obj3 := getTestObjectInfo(3, getOID(3), joinVers(obj1, obj2), "", "*")
obj4 := getTestObjectInfo(4, getOID(4), joinVers(obj1, obj2), obj2.Version(), obj2.Version())
obj5 := getTestObjectInfo(5, getOID(5), obj1.Version(), obj1.Version(), obj1.Version())
obj6 := getTestObjectInfo(6, getOID(6), joinVers(obj1, obj2, obj3), obj3.Version(), obj3.Version())
for _, tc := range []struct {
versions *objectVersions
expected *ObjectInfo
}{
{
versions: &objectVersions{},
expected: nil,
},
{
versions: &objectVersions{
objects: []*ObjectInfo{obj2, obj1},
addList: []string{obj1.Version(), obj2.Version()},
},
expected: obj2,
},
{
versions: &objectVersions{
objects: []*ObjectInfo{obj2, obj1, obj3},
addList: []string{obj1.Version(), obj2.Version(), obj3.Version()},
},
expected: nil,
},
{
versions: &objectVersions{
objects: []*ObjectInfo{obj2, obj1, obj4},
addList: []string{obj1.Version(), obj2.Version(), obj4.Version()},
delList: []string{obj2.Version()},
},
expected: obj1,
},
{
versions: &objectVersions{
objects: []*ObjectInfo{obj1, obj5},
addList: []string{obj1.Version(), obj5.Version()},
delList: []string{obj1.Version()},
},
expected: nil,
},
{
versions: &objectVersions{
objects: []*ObjectInfo{obj5},
},
expected: nil,
},
{
versions: &objectVersions{
objects: []*ObjectInfo{obj1, obj2, obj3, obj6},
addList: []string{obj1.Version(), obj2.Version(), obj3.Version(), obj6.Version()},
delList: []string{obj3.Version()},
},
expected: obj2,
},
{
versions: &objectVersions{
objects: []*ObjectInfo{obj1, obj1V2},
addList: []string{obj1.Version(), obj1V2.Version()},
},
// creation epochs are equal
// obj1 version/oid > obj1_1 version/oid
expected: obj1,
},
} {
actualObjInfo := tc.versions.getLast()
require.Equal(t, tc.expected, actualObjInfo)
}
}
func TestAppendVersions(t *testing.T) {
obj1 := getTestObjectInfo(1, getOID(1), "", "", "")
obj2 := getTestObjectInfo(2, getOID(2), obj1.Version(), "", "")
obj3 := getTestObjectInfo(3, getOID(3), joinVers(obj1, obj2), "", "*")
obj4 := getTestObjectInfo(4, getOID(4), joinVers(obj1, obj2), obj2.Version(), obj2.Version())
for _, tc := range []struct {
versions *objectVersions
objectToAdd *ObjectInfo
expectedVersions *objectVersions
}{
{
versions: &objectVersions{},
objectToAdd: obj1,
expectedVersions: &objectVersions{
objects: []*ObjectInfo{obj1},
addList: []string{obj1.Version()},
},
},
{
versions: &objectVersions{objects: []*ObjectInfo{obj1}},
objectToAdd: obj2,
expectedVersions: &objectVersions{
objects: []*ObjectInfo{obj1, obj2},
addList: []string{obj1.Version(), obj2.Version()},
},
},
{
versions: &objectVersions{objects: []*ObjectInfo{obj1, obj2}},
objectToAdd: obj3,
expectedVersions: &objectVersions{
objects: []*ObjectInfo{obj1, obj2, obj3},
addList: []string{obj1.Version(), obj2.Version(), obj3.Version()},
},
},
{
versions: &objectVersions{objects: []*ObjectInfo{obj1, obj2}},
objectToAdd: obj4,
expectedVersions: &objectVersions{
objects: []*ObjectInfo{obj1, obj2, obj4},
addList: []string{obj1.Version(), obj2.Version(), obj4.Version()},
delList: []string{obj2.Version()},
},
},
} {
tc.versions.appendVersion(tc.objectToAdd)
require.Equal(t, tc.expectedVersions, tc.versions)
}
}
func joinVers(objs ...*ObjectInfo) string {
if len(objs) == 0 {
return ""
}
var versions []string
for _, obj := range objs {
versions = append(versions, obj.Version())
}
return strings.Join(versions, ",")
}
func getOID(id byte) *object.ID {
b := make([]byte, 32)
b[0] = id
oid := object.NewID()
oid.SetSHA256(sha256.Sum256(b))
return oid
}
func getTestObjectInfo(epoch uint64, oid *object.ID, addAttr, delAttr, delMarkAttr string) *ObjectInfo {
headers := make(map[string]string)
if addAttr != "" {
headers[versionsAddAttr] = addAttr
}
if delAttr != "" {
headers[versionsDelAttr] = delAttr
}
if delMarkAttr != "" {
headers[versionsDeleteMarkAttr] = delMarkAttr
}
return &ObjectInfo{
id: oid,
CreationEpoch: epoch,
Headers: headers,
}
}