forked from TrueCloudLab/frostfs-s3-gw
parent
9fb3fb1274
commit
1a456eaa8b
12 changed files with 4 additions and 647 deletions
|
@ -81,15 +81,6 @@ func (b *BucketInfo) NotificationConfigurationObjectName() string {
|
||||||
// Version returns object version from ObjectInfo.
|
// Version returns object version from ObjectInfo.
|
||||||
func (o *ObjectInfo) Version() string { return o.ID.EncodeToString() }
|
func (o *ObjectInfo) Version() string { return o.ID.EncodeToString() }
|
||||||
|
|
||||||
// NullableVersion returns object version from ObjectInfo.
|
|
||||||
// Return "null" if "S3-Versions-unversioned" header is present.
|
|
||||||
func (o *ObjectInfo) NullableVersion() string {
|
|
||||||
if _, ok := o.Headers["S3-Versions-unversioned"]; ok {
|
|
||||||
return "null"
|
|
||||||
}
|
|
||||||
return o.Version()
|
|
||||||
}
|
|
||||||
|
|
||||||
// NiceName returns object name for cache.
|
// NiceName returns object name for cache.
|
||||||
func (o *ObjectInfo) NiceName() string { return o.Bucket + "/" + o.Name }
|
func (o *ObjectInfo) NiceName() string { return o.Bucket + "/" + o.Name }
|
||||||
|
|
||||||
|
@ -101,9 +92,3 @@ func (o *ObjectInfo) Address() oid.Address {
|
||||||
|
|
||||||
return addr
|
return addr
|
||||||
}
|
}
|
||||||
|
|
||||||
// LegalHoldObject returns the name of a system object for a lock object.
|
|
||||||
func (o *ObjectInfo) LegalHoldObject() string { return ".lock." + o.Name + "." + o.Version() }
|
|
||||||
|
|
||||||
// RetentionObject returns the name of a system object for a retention lock object.
|
|
||||||
func (o *ObjectInfo) RetentionObject() string { return ".retention." + o.Name + "." + o.Version() }
|
|
||||||
|
|
|
@ -115,7 +115,7 @@ func writeAttributesHeaders(h http.Header, info *data.ObjectInfo, params *GetObj
|
||||||
h.Set(api.AmzVersionID, info.Version())
|
h.Set(api.AmzVersionID, info.Version())
|
||||||
}
|
}
|
||||||
|
|
||||||
if _, ok := info.Headers[layer.VersionsDeleteMarkAttr]; ok {
|
if info.IsDeleteMarker {
|
||||||
h.Set(api.AmzDeleteMarker, strconv.FormatBool(true))
|
h.Set(api.AmzDeleteMarker, strconv.FormatBool(true))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -9,7 +9,6 @@ import (
|
||||||
|
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
"github.com/nspcc-dev/neofs-s3-gw/api"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
"github.com/nspcc-dev/neofs-s3-gw/api/errors"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/session"
|
"github.com/nspcc-dev/neofs-sdk-go/session"
|
||||||
|
@ -275,16 +274,6 @@ func (h *handler) UploadPartCopy(w http.ResponseWriter, r *http.Request) {
|
||||||
h.logAndSendError(w, "could not head source object", reqInfo, err, additional...)
|
h.logAndSendError(w, "could not head source object", reqInfo, err, additional...)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if isDeleted(srcInfo) {
|
|
||||||
if versionID != "" {
|
|
||||||
h.logAndSendError(w, "could not head source object version", reqInfo,
|
|
||||||
errors.GetAPIError(errors.ErrBadRequest), additional...)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
h.logAndSendError(w, "could not head source object", reqInfo,
|
|
||||||
errors.GetAPIError(errors.ErrNoSuchKey), additional...)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
args, err := parseCopyObjectArgs(r.Header)
|
args, err := parseCopyObjectArgs(r.Header)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -423,11 +412,6 @@ func (h *handler) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.
|
||||||
h.logAndSendError(w, "could not get bucket settings", reqInfo, err)
|
h.logAndSendError(w, "could not get bucket settings", reqInfo, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err = h.obj.DeleteSystemObject(r.Context(), bktInfo, layer.FormUploadPartName(uploadID, uploadInfo.Key, 0)); err != nil {
|
|
||||||
h.logAndSendError(w, "could not delete init file of multipart upload", reqInfo, err, additional...)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
response := CompleteMultipartUploadResponse{
|
response := CompleteMultipartUploadResponse{
|
||||||
Bucket: objInfo.Bucket,
|
Bucket: objInfo.Bucket,
|
||||||
ETag: objInfo.HashSum,
|
ETag: objInfo.HashSum,
|
||||||
|
@ -633,7 +617,3 @@ func encodeListPartsToResponse(info *layer.ListPartsInfo, params *layer.ListPart
|
||||||
Parts: info.Parts,
|
Parts: info.Parts,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func isDeleted(objInfo *data.ObjectInfo) bool {
|
|
||||||
return objInfo.Headers[layer.VersionsDeleteMarkAttr] == layer.DelMarkFullObject
|
|
||||||
}
|
|
||||||
|
|
|
@ -281,7 +281,7 @@ func encodeListObjectVersionsToResponse(info *layer.ListObjectVersionsInfo, buck
|
||||||
DisplayName: ver.Object.Owner.String(),
|
DisplayName: ver.Object.Owner.String(),
|
||||||
},
|
},
|
||||||
Size: ver.Object.Size,
|
Size: ver.Object.Size,
|
||||||
VersionID: ver.Object.NullableVersion(),
|
VersionID: ver.Object.Version(), // todo return "null" version for unversioned https://github.com/nspcc-dev/neofs-s3-gw/issues/474
|
||||||
ETag: ver.Object.HashSum,
|
ETag: ver.Object.HashSum,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
|
@ -237,7 +237,6 @@ type (
|
||||||
ListObjectVersions(ctx context.Context, p *ListObjectVersionsParams) (*ListObjectVersionsInfo, error)
|
ListObjectVersions(ctx context.Context, p *ListObjectVersionsParams) (*ListObjectVersionsInfo, error)
|
||||||
|
|
||||||
DeleteObjects(ctx context.Context, p *DeleteObjectParams) ([]*VersionedObject, error)
|
DeleteObjects(ctx context.Context, p *DeleteObjectParams) ([]*VersionedObject, error)
|
||||||
DeleteSystemObject(ctx context.Context, bktInfo *data.BucketInfo, name string) error
|
|
||||||
|
|
||||||
CreateMultipartUpload(ctx context.Context, p *CreateMultipartParams) error
|
CreateMultipartUpload(ctx context.Context, p *CreateMultipartParams) error
|
||||||
CompleteMultipartUpload(ctx context.Context, p *CompleteMultipartParams) (*UploadData, *data.ObjectInfo, error)
|
CompleteMultipartUpload(ctx context.Context, p *CompleteMultipartParams) (*UploadData, *data.ObjectInfo, error)
|
||||||
|
|
|
@ -22,9 +22,7 @@ import (
|
||||||
const (
|
const (
|
||||||
UploadIDAttributeName = "S3-Upload-Id"
|
UploadIDAttributeName = "S3-Upload-Id"
|
||||||
UploadPartNumberAttributeName = "S3-Upload-Part-Number"
|
UploadPartNumberAttributeName = "S3-Upload-Part-Number"
|
||||||
UploadKeyAttributeName = "S3-Upload-Key"
|
|
||||||
UploadCompletedParts = "S3-Completed-Parts"
|
UploadCompletedParts = "S3-Completed-Parts"
|
||||||
UploadPartKeyPrefix = ".upload-"
|
|
||||||
|
|
||||||
metaPrefix = "meta-"
|
metaPrefix = "meta-"
|
||||||
aclPrefix = "acl-"
|
aclPrefix = "acl-"
|
||||||
|
@ -254,9 +252,6 @@ func (n *layer) UploadPartCopy(ctx context.Context, p *UploadCopyParams) (*data.
|
||||||
return nil, errors.GetAPIError(errors.ErrEntityTooLarge)
|
return nil, errors.GetAPIError(errors.ErrEntityTooLarge)
|
||||||
}
|
}
|
||||||
|
|
||||||
metadata := make(map[string]string)
|
|
||||||
appendUploadHeaders(metadata, p.Info.UploadID, p.Info.Key, p.PartNumber)
|
|
||||||
|
|
||||||
pr, pw := io.Pipe()
|
pr, pw := io.Pipe()
|
||||||
|
|
||||||
go func() {
|
go func() {
|
||||||
|
@ -571,10 +566,6 @@ func (n *layer) getUploadParts(ctx context.Context, p *UploadInfoParams) (*data.
|
||||||
return multipartInfo, res, nil
|
return multipartInfo, res, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func FormUploadPartName(uploadID, key string, partNumber int) string {
|
|
||||||
return UploadPartKeyPrefix + uploadID + "-" + key + "-" + strconv.Itoa(partNumber)
|
|
||||||
}
|
|
||||||
|
|
||||||
func trimAfterUploadIDAndKey(key, id string, uploads []*UploadInfo) []*UploadInfo {
|
func trimAfterUploadIDAndKey(key, id string, uploads []*UploadInfo) []*UploadInfo {
|
||||||
var res []*UploadInfo
|
var res []*UploadInfo
|
||||||
if len(uploads) != 0 && uploads[len(uploads)-1].Key < key {
|
if len(uploads) != 0 && uploads[len(uploads)-1].Key < key {
|
||||||
|
@ -630,8 +621,3 @@ func uploadInfoFromMultipartInfo(uploadInfo *data.MultipartInfo, prefix, delimit
|
||||||
Created: uploadInfo.Created,
|
Created: uploadInfo.Created,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func appendUploadHeaders(metadata map[string]string, uploadID, key string, partNumber int) {
|
|
||||||
metadata[UploadIDAttributeName] = uploadID
|
|
||||||
metadata[UploadPartNumberAttributeName] = strconv.Itoa(partNumber)
|
|
||||||
}
|
|
||||||
|
|
|
@ -27,6 +27,8 @@ type TestNeoFS struct {
|
||||||
currentEpoch uint64
|
currentEpoch uint64
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const objectSystemAttributeName = "S3-System-name"
|
||||||
|
|
||||||
func NewTestNeoFS() *TestNeoFS {
|
func NewTestNeoFS() *TestNeoFS {
|
||||||
return &TestNeoFS{
|
return &TestNeoFS{
|
||||||
objects: make(map[string]*object.Object),
|
objects: make(map[string]*object.Object),
|
||||||
|
|
|
@ -26,12 +26,6 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
type (
|
type (
|
||||||
findParams struct {
|
|
||||||
attr [2]string
|
|
||||||
bkt *data.BucketInfo
|
|
||||||
prefix string
|
|
||||||
}
|
|
||||||
|
|
||||||
getParams struct {
|
getParams struct {
|
||||||
// payload range
|
// payload range
|
||||||
off, ln uint64
|
off, ln uint64
|
||||||
|
@ -70,29 +64,6 @@ type (
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
func (n *layer) objectSearchByName(ctx context.Context, bktInfo *data.BucketInfo, filename string) ([]oid.ID, error) {
|
|
||||||
f := &findParams{
|
|
||||||
attr: [2]string{object.AttributeFileName, filename},
|
|
||||||
bkt: bktInfo,
|
|
||||||
}
|
|
||||||
return n.objectSearch(ctx, f)
|
|
||||||
}
|
|
||||||
|
|
||||||
// objectSearch returns all available objects by search params.
|
|
||||||
func (n *layer) objectSearch(ctx context.Context, p *findParams) ([]oid.ID, error) {
|
|
||||||
prm := PrmObjectSelect{
|
|
||||||
Container: p.bkt.CID,
|
|
||||||
ExactAttribute: p.attr,
|
|
||||||
FilePrefix: p.prefix,
|
|
||||||
}
|
|
||||||
|
|
||||||
n.prepareAuthParameters(ctx, &prm.PrmAuth, p.bkt.Owner)
|
|
||||||
|
|
||||||
res, err := n.neoFS.SelectObjects(ctx, prm)
|
|
||||||
|
|
||||||
return res, n.transformNeofsError(ctx, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
func newAddress(cnr cid.ID, obj oid.ID) oid.Address {
|
func newAddress(cnr cid.ID, obj oid.ID) oid.Address {
|
||||||
var addr oid.Address
|
var addr oid.Address
|
||||||
addr.SetContainer(cnr)
|
addr.SetContainer(cnr)
|
||||||
|
@ -303,44 +274,6 @@ func (n *layer) headLastVersionIfNotDeleted(ctx context.Context, bkt *data.Bucke
|
||||||
return objInfo, nil
|
return objInfo, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *layer) headVersions(ctx context.Context, bkt *data.BucketInfo, objectName string) (*objectVersions, error) {
|
|
||||||
ids, err := n.objectSearchByName(ctx, bkt, objectName)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
versions := newObjectVersions(objectName)
|
|
||||||
if len(ids) == 0 {
|
|
||||||
return versions, apiErrors.GetAPIError(apiErrors.ErrNoSuchKey)
|
|
||||||
}
|
|
||||||
|
|
||||||
for i := range ids {
|
|
||||||
meta, err := n.objectHead(ctx, bkt, ids[i])
|
|
||||||
if err != nil {
|
|
||||||
n.log.Warn("couldn't head object",
|
|
||||||
zap.Stringer("object id", &ids[i]),
|
|
||||||
zap.Stringer("bucket id", bkt.CID),
|
|
||||||
zap.Error(err))
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if err = n.objCache.Put(*meta); err != nil {
|
|
||||||
n.log.Warn("couldn't put meta to objects cache",
|
|
||||||
zap.Stringer("object id", &ids[i]),
|
|
||||||
zap.Stringer("bucket id", bkt.CID),
|
|
||||||
zap.Error(err))
|
|
||||||
}
|
|
||||||
|
|
||||||
if oi := objInfoFromMeta(bkt, meta); oi != nil {
|
|
||||||
if isSystem(oi) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
versions.appendVersion(oi)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return versions, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (n *layer) headVersion(ctx context.Context, bkt *data.BucketInfo, p *HeadObjectParams) (*data.ObjectInfo, error) {
|
func (n *layer) headVersion(ctx context.Context, bkt *data.BucketInfo, p *HeadObjectParams) (*data.ObjectInfo, error) {
|
||||||
var err error
|
var err error
|
||||||
var foundVersion *data.NodeVersion
|
var foundVersion *data.NodeVersion
|
||||||
|
@ -530,10 +463,6 @@ func (n *layer) getLatestObjectsVersions(ctx context.Context, bkt *data.BucketIn
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if oi := objectInfoFromMeta(bkt, obj, prefix, delimiter); oi != nil {
|
if oi := objectInfoFromMeta(bkt, obj, prefix, delimiter); oi != nil {
|
||||||
if isSystem(oi) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
objectsMap[oi.Name] = oi
|
objectsMap[oi.Name] = oi
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -601,19 +530,6 @@ func (n *layer) getAllObjectsVersions(ctx context.Context, bkt *data.BucketInfo,
|
||||||
return versions, nil
|
return versions, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func splitVersions(header string) []string {
|
|
||||||
if len(header) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
return strings.Split(header, ",")
|
|
||||||
}
|
|
||||||
|
|
||||||
func isSystem(obj *data.ObjectInfo) bool {
|
|
||||||
return len(obj.Headers[objectSystemAttributeName]) > 0 ||
|
|
||||||
len(obj.Headers[attrVersionsIgnore]) > 0
|
|
||||||
}
|
|
||||||
|
|
||||||
func IsSystemHeader(key string) bool {
|
func IsSystemHeader(key string) bool {
|
||||||
return strings.HasPrefix(key, "S3-")
|
return strings.HasPrefix(key, "S3-")
|
||||||
}
|
}
|
||||||
|
|
|
@ -129,26 +129,6 @@ func (n *layer) GetLockInfo(ctx context.Context, objVersion *ObjectVersion) (*da
|
||||||
return lockInfo, nil
|
return lockInfo, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *layer) DeleteSystemObject(ctx context.Context, bktInfo *data.BucketInfo, name string) error {
|
|
||||||
f := &findParams{
|
|
||||||
attr: [2]string{objectSystemAttributeName, name},
|
|
||||||
bkt: bktInfo,
|
|
||||||
}
|
|
||||||
ids, err := n.objectSearch(ctx, f)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
n.systemCache.Delete(systemObjectKey(bktInfo, name))
|
|
||||||
for i := range ids {
|
|
||||||
if err = n.objectDelete(ctx, bktInfo, ids[i]); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (n *layer) getCORS(ctx context.Context, bkt *data.BucketInfo, sysName string) (*data.CORSConfiguration, error) {
|
func (n *layer) getCORS(ctx context.Context, bkt *data.BucketInfo, sysName string) (*data.CORSConfiguration, error) {
|
||||||
if cors := n.systemCache.GetCORS(systemObjectKey(bkt, sysName)); cors != nil {
|
if cors := n.systemCache.GetCORS(systemObjectKey(bkt, sysName)); cors != nil {
|
||||||
return cors, nil
|
return cors, nil
|
||||||
|
@ -184,43 +164,6 @@ func (n *layer) getCORS(ctx context.Context, bkt *data.BucketInfo, sysName strin
|
||||||
return cors, nil
|
return cors, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *layer) headSystemVersions(ctx context.Context, bkt *data.BucketInfo, sysName string) (*objectVersions, error) {
|
|
||||||
f := &findParams{
|
|
||||||
attr: [2]string{objectSystemAttributeName, sysName},
|
|
||||||
bkt: bkt,
|
|
||||||
}
|
|
||||||
ids, err := n.objectSearch(ctx, f)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
versions := newObjectVersions(sysName)
|
|
||||||
for i := range ids {
|
|
||||||
meta, err := n.objectHead(ctx, bkt, ids[i])
|
|
||||||
if err != nil {
|
|
||||||
n.log.Warn("couldn't head object",
|
|
||||||
zap.Stringer("object id", &ids[i]),
|
|
||||||
zap.Stringer("bucket id", bkt.CID),
|
|
||||||
zap.Error(err))
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if oi := objInfoFromMeta(bkt, meta); oi != nil {
|
|
||||||
if !isSystem(oi) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
versions.appendVersion(oi)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
lastVersion := versions.getLast()
|
|
||||||
if lastVersion == nil {
|
|
||||||
return nil, errors.GetAPIError(errors.ErrNoSuchKey)
|
|
||||||
}
|
|
||||||
|
|
||||||
return versions, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// systemObjectKey is a key to use in SystemCache.
|
// systemObjectKey is a key to use in SystemCache.
|
||||||
func systemObjectKey(bktInfo *data.BucketInfo, obj string) string {
|
func systemObjectKey(bktInfo *data.BucketInfo, obj string) string {
|
||||||
return bktInfo.Name + obj
|
return bktInfo.Name + obj
|
||||||
|
|
|
@ -161,7 +161,3 @@ func GetBoxData(ctx context.Context) (*accessbox.Box, error) {
|
||||||
}
|
}
|
||||||
return boxData, nil
|
return boxData, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func formBucketTagObjectName(name string) string {
|
|
||||||
return ".tagset." + name
|
|
||||||
}
|
|
||||||
|
|
|
@ -2,220 +2,15 @@ package layer
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"math"
|
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
|
||||||
|
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
"github.com/nspcc-dev/neofs-s3-gw/api/data"
|
||||||
)
|
)
|
||||||
|
|
||||||
type objectVersions struct {
|
|
||||||
name string
|
|
||||||
objects []*data.ObjectInfo
|
|
||||||
addList []string
|
|
||||||
delList []string
|
|
||||||
isSorted bool
|
|
||||||
}
|
|
||||||
|
|
||||||
func FromUnversioned() VersionOption {
|
|
||||||
return func(options *versionOptions) {
|
|
||||||
options.unversioned = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type VersionOption func(*versionOptions)
|
|
||||||
|
|
||||||
type versionOptions struct {
|
|
||||||
unversioned bool
|
|
||||||
}
|
|
||||||
|
|
||||||
func formVersionOptions(opts ...VersionOption) *versionOptions {
|
|
||||||
options := &versionOptions{}
|
|
||||||
for _, opt := range opts {
|
|
||||||
opt(options)
|
|
||||||
}
|
|
||||||
return options
|
|
||||||
}
|
|
||||||
|
|
||||||
const (
|
const (
|
||||||
VersionsDeleteMarkAttr = "S3-Versions-delete-mark"
|
|
||||||
DelMarkFullObject = "*"
|
|
||||||
|
|
||||||
unversionedObjectVersionID = "null"
|
unversionedObjectVersionID = "null"
|
||||||
objectSystemAttributeName = "S3-System-name"
|
|
||||||
attrVersionsIgnore = "S3-Versions-ignore"
|
|
||||||
versionsDelAttr = "S3-Versions-del"
|
|
||||||
versionsAddAttr = "S3-Versions-add"
|
|
||||||
versionsUnversionedAttr = "S3-Versions-unversioned"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func newObjectVersions(name string) *objectVersions {
|
|
||||||
return &objectVersions{name: name}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (v *objectVersions) appendVersion(oi *data.ObjectInfo) {
|
|
||||||
delVers := splitVersions(oi.Headers[versionsDelAttr])
|
|
||||||
v.objects = append(v.objects, oi)
|
|
||||||
|
|
||||||
for _, del := range delVers {
|
|
||||||
if !contains(v.delList, del) {
|
|
||||||
v.delList = append(v.delList, del)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
v.isSorted = false
|
|
||||||
}
|
|
||||||
|
|
||||||
func (v *objectVersions) sort() {
|
|
||||||
if !v.isSorted {
|
|
||||||
sort.Slice(v.objects, func(i, j int) bool {
|
|
||||||
o1, o2 := v.objects[i], v.objects[j]
|
|
||||||
if o1.CreationEpoch == o2.CreationEpoch {
|
|
||||||
l1, l2 := o1.Headers[versionsAddAttr], o2.Headers[versionsAddAttr]
|
|
||||||
if len(l1) != len(l2) {
|
|
||||||
if strings.HasPrefix(l1, l2) {
|
|
||||||
return false
|
|
||||||
} else if strings.HasPrefix(l2, l1) {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return o1.Version() < o2.Version()
|
|
||||||
}
|
|
||||||
return o1.CreationEpoch < o2.CreationEpoch
|
|
||||||
})
|
|
||||||
|
|
||||||
v.formAddList()
|
|
||||||
v.isSorted = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (v *objectVersions) formAddList() {
|
|
||||||
for i := 0; i < len(v.objects); i++ {
|
|
||||||
var conflicts [][]string
|
|
||||||
for { // forming conflicts set (objects with the same creation epoch)
|
|
||||||
addVers := append(splitVersions(v.objects[i].Headers[versionsAddAttr]), v.objects[i].Version())
|
|
||||||
conflicts = append(conflicts, addVers)
|
|
||||||
if i == len(v.objects)-1 || v.objects[i].CreationEpoch != v.objects[i+1].CreationEpoch ||
|
|
||||||
containsVersions(v.objects[i+1], addVers) {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
i++
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(conflicts) == 1 {
|
|
||||||
v.addList = addIfNotContains(v.addList, conflicts[0])
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
commonVersions, prevConflictedVersions, conflictedVersions := mergeVersionsConflicts(conflicts)
|
|
||||||
v.addList = commonVersions
|
|
||||||
v.addList = addIfNotContains(v.addList, prevConflictedVersions)
|
|
||||||
v.addList = addIfNotContains(v.addList, conflictedVersions)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func containsVersions(obj *data.ObjectInfo, versions []string) bool {
|
|
||||||
header := obj.Headers[versionsAddAttr]
|
|
||||||
for _, version := range versions {
|
|
||||||
if !strings.Contains(header, version) {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
func addIfNotContains(list1, list2 []string) []string {
|
|
||||||
for _, add := range list2 {
|
|
||||||
if !contains(list1, add) {
|
|
||||||
list1 = append(list1, add)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return list1
|
|
||||||
}
|
|
||||||
|
|
||||||
func mergeVersionsConflicts(conflicts [][]string) ([]string, []string, []string) {
|
|
||||||
var currentVersions []string
|
|
||||||
var prevVersions []string
|
|
||||||
minLength := math.MaxInt32
|
|
||||||
for _, conflicted := range conflicts {
|
|
||||||
if len(conflicted)-1 < minLength {
|
|
||||||
minLength = len(conflicted) - 1
|
|
||||||
}
|
|
||||||
// last := conflicted[len(conflicted)-1]
|
|
||||||
// conflicts[j] = conflicted[:len(conflicted)-1]
|
|
||||||
// currentVersions = append(currentVersions, last)
|
|
||||||
}
|
|
||||||
var commonAddedVersions []string
|
|
||||||
diffIndex := 0
|
|
||||||
LOOP:
|
|
||||||
for k := 0; k < minLength; k++ {
|
|
||||||
candidate := conflicts[0][k]
|
|
||||||
for _, conflicted := range conflicts {
|
|
||||||
if conflicted[k] != candidate {
|
|
||||||
diffIndex = k
|
|
||||||
break LOOP
|
|
||||||
}
|
|
||||||
}
|
|
||||||
commonAddedVersions = append(commonAddedVersions, candidate)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, conflicted := range conflicts {
|
|
||||||
for j := diffIndex; j < len(conflicted); j++ {
|
|
||||||
prevVersions = append(prevVersions, conflicted[j])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
sort.Strings(prevVersions)
|
|
||||||
sort.Strings(currentVersions)
|
|
||||||
return commonAddedVersions, prevVersions, currentVersions
|
|
||||||
}
|
|
||||||
|
|
||||||
func (v *objectVersions) isEmpty() bool {
|
|
||||||
return v == nil || len(v.objects) == 0
|
|
||||||
}
|
|
||||||
|
|
||||||
func (v *objectVersions) getLast(opts ...VersionOption) *data.ObjectInfo {
|
|
||||||
if v.isEmpty() {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
options := formVersionOptions(opts...)
|
|
||||||
|
|
||||||
v.sort()
|
|
||||||
existedVersions := v.existedVersions()
|
|
||||||
for i := len(v.objects) - 1; i >= 0; i-- {
|
|
||||||
if contains(existedVersions, v.objects[i].Version()) {
|
|
||||||
delMarkHeader := v.objects[i].Headers[VersionsDeleteMarkAttr]
|
|
||||||
if delMarkHeader == "" {
|
|
||||||
if options.unversioned && v.objects[i].Headers[versionsUnversionedAttr] != "true" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
return v.objects[i]
|
|
||||||
}
|
|
||||||
if delMarkHeader == DelMarkFullObject {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (v *objectVersions) existedVersions() []string {
|
|
||||||
v.sort()
|
|
||||||
var res []string
|
|
||||||
for _, add := range v.addList {
|
|
||||||
if !contains(v.delList, add) {
|
|
||||||
res = append(res, add)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return res
|
|
||||||
}
|
|
||||||
|
|
||||||
func (v *objectVersions) getAddHeader() string {
|
|
||||||
v.sort()
|
|
||||||
return strings.Join(v.addList, ",")
|
|
||||||
}
|
|
||||||
|
|
||||||
func (n *layer) ListObjectVersions(ctx context.Context, p *ListObjectVersionsParams) (*ListObjectVersionsInfo, error) {
|
func (n *layer) ListObjectVersions(ctx context.Context, p *ListObjectVersionsParams) (*ListObjectVersionsInfo, error) {
|
||||||
var (
|
var (
|
||||||
allObjects = make([]*data.ObjectInfo, 0, p.MaxKeys)
|
allObjects = make([]*data.ObjectInfo, 0, p.MaxKeys)
|
||||||
|
@ -293,12 +88,3 @@ func triageVersions(objVersions []*ObjectVersionInfo) ([]*ObjectVersionInfo, []*
|
||||||
|
|
||||||
return resVersion, resDelMarkVersions
|
return resVersion, resDelMarkVersions
|
||||||
}
|
}
|
||||||
|
|
||||||
func contains(list []string, elem string) bool {
|
|
||||||
for _, item := range list {
|
|
||||||
if elem == item {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
|
@ -3,8 +3,6 @@ package layer
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||||
|
@ -115,17 +113,6 @@ func (tc *testContext) checkListObjects(ids ...oid.ID) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (tc *testContext) getSystemObject(objectName string) *object.Object {
|
|
||||||
for _, obj := range tc.testNeoFS.Objects() {
|
|
||||||
for _, attr := range obj.Attributes() {
|
|
||||||
if attr.Key() == objectSystemAttributeName && attr.Value() == objectName {
|
|
||||||
return obj
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (tc *testContext) getObjectByID(objID oid.ID) *object.Object {
|
func (tc *testContext) getObjectByID(objID oid.ID) *object.Object {
|
||||||
for _, obj := range tc.testNeoFS.Objects() {
|
for _, obj := range tc.testNeoFS.Objects() {
|
||||||
id, _ := obj.ID()
|
id, _ := obj.ID()
|
||||||
|
@ -302,80 +289,6 @@ func TestVersioningDeleteSpecificObjectVersion(t *testing.T) {
|
||||||
require.Equal(t, objV3Info.Version(), resInfo.Version())
|
require.Equal(t, objV3Info.Version(), resInfo.Version())
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestGetLastVersion(t *testing.T) {
|
|
||||||
obj1 := getTestObjectInfo(1, "", "", "")
|
|
||||||
obj1V2 := getTestObjectInfo(2, "", "", "")
|
|
||||||
obj2 := getTestObjectInfoEpoch(1, 2, obj1.Version(), "", "")
|
|
||||||
obj3 := getTestObjectInfoEpoch(1, 3, joinVers(obj1, obj2), "", "*")
|
|
||||||
obj4 := getTestObjectInfoEpoch(1, 4, joinVers(obj1, obj2), obj2.Version(), obj2.Version())
|
|
||||||
obj5 := getTestObjectInfoEpoch(1, 5, obj1.Version(), obj1.Version(), obj1.Version())
|
|
||||||
obj6 := getTestObjectInfoEpoch(1, 6, joinVers(obj1, obj2, obj3), obj3.Version(), obj3.Version())
|
|
||||||
|
|
||||||
for _, tc := range []struct {
|
|
||||||
versions *objectVersions
|
|
||||||
expected *data.ObjectInfo
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
versions: &objectVersions{},
|
|
||||||
expected: nil,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
versions: &objectVersions{
|
|
||||||
objects: []*data.ObjectInfo{obj2, obj1},
|
|
||||||
addList: []string{obj1.Version(), obj2.Version()},
|
|
||||||
},
|
|
||||||
expected: obj2,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
versions: &objectVersions{
|
|
||||||
objects: []*data.ObjectInfo{obj2, obj1, obj3},
|
|
||||||
addList: []string{obj1.Version(), obj2.Version(), obj3.Version()},
|
|
||||||
},
|
|
||||||
expected: nil,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
versions: &objectVersions{
|
|
||||||
objects: []*data.ObjectInfo{obj2, obj1, obj4},
|
|
||||||
addList: []string{obj1.Version(), obj2.Version(), obj4.Version()},
|
|
||||||
delList: []string{obj2.Version()},
|
|
||||||
},
|
|
||||||
expected: obj1,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
versions: &objectVersions{
|
|
||||||
objects: []*data.ObjectInfo{obj1, obj5},
|
|
||||||
addList: []string{obj1.Version(), obj5.Version()},
|
|
||||||
delList: []string{obj1.Version()},
|
|
||||||
},
|
|
||||||
expected: nil,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
versions: &objectVersions{
|
|
||||||
objects: []*data.ObjectInfo{obj5},
|
|
||||||
},
|
|
||||||
expected: nil,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
versions: &objectVersions{
|
|
||||||
objects: []*data.ObjectInfo{obj1, obj2, obj3, obj6},
|
|
||||||
addList: []string{obj1.Version(), obj2.Version(), obj3.Version(), obj6.Version()},
|
|
||||||
delList: []string{obj3.Version()},
|
|
||||||
},
|
|
||||||
expected: obj2,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
versions: &objectVersions{
|
|
||||||
objects: []*data.ObjectInfo{obj1, obj1V2},
|
|
||||||
addList: []string{obj1.Version(), obj1V2.Version()},
|
|
||||||
},
|
|
||||||
expected: obj1V2,
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
actualObjInfo := tc.versions.getLast()
|
|
||||||
require.Equal(t, tc.expected, actualObjInfo)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNoVersioningDeleteObject(t *testing.T) {
|
func TestNoVersioningDeleteObject(t *testing.T) {
|
||||||
tc := prepareContext(t)
|
tc := prepareContext(t)
|
||||||
|
|
||||||
|
@ -389,152 +302,3 @@ func TestNoVersioningDeleteObject(t *testing.T) {
|
||||||
tc.getObject(tc.obj, "", true)
|
tc.getObject(tc.obj, "", true)
|
||||||
tc.checkListObjects()
|
tc.checkListObjects()
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAppendVersions(t *testing.T) {
|
|
||||||
obj1 := getTestObjectInfo(1, "", "", "")
|
|
||||||
obj2 := getTestObjectInfo(2, obj1.Version(), "", "")
|
|
||||||
obj3 := getTestObjectInfo(3, joinVers(obj1, obj2), "", "*")
|
|
||||||
obj4 := getTestObjectInfo(4, joinVers(obj1, obj2), obj2.Version(), obj2.Version())
|
|
||||||
obj5 := getTestObjectInfo(5, joinVers(obj1, obj2), "", "")
|
|
||||||
obj6 := getTestObjectInfo(6, joinVers(obj1, obj3), "", "")
|
|
||||||
|
|
||||||
for _, tc := range []struct {
|
|
||||||
versions *objectVersions
|
|
||||||
objectToAdd *data.ObjectInfo
|
|
||||||
expectedVersions *objectVersions
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
versions: &objectVersions{},
|
|
||||||
objectToAdd: obj1,
|
|
||||||
expectedVersions: &objectVersions{
|
|
||||||
objects: []*data.ObjectInfo{obj1},
|
|
||||||
addList: []string{obj1.Version()},
|
|
||||||
isSorted: true,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
versions: &objectVersions{objects: []*data.ObjectInfo{obj1}},
|
|
||||||
objectToAdd: obj2,
|
|
||||||
expectedVersions: &objectVersions{
|
|
||||||
objects: []*data.ObjectInfo{obj1, obj2},
|
|
||||||
addList: []string{obj1.Version(), obj2.Version()},
|
|
||||||
isSorted: true,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
versions: &objectVersions{objects: []*data.ObjectInfo{obj1, obj2}},
|
|
||||||
objectToAdd: obj3,
|
|
||||||
expectedVersions: &objectVersions{
|
|
||||||
objects: []*data.ObjectInfo{obj1, obj2, obj3},
|
|
||||||
addList: []string{obj1.Version(), obj2.Version(), obj3.Version()},
|
|
||||||
isSorted: true,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
versions: &objectVersions{objects: []*data.ObjectInfo{obj1, obj2}},
|
|
||||||
objectToAdd: obj4,
|
|
||||||
expectedVersions: &objectVersions{
|
|
||||||
objects: []*data.ObjectInfo{obj1, obj2, obj4},
|
|
||||||
addList: []string{obj1.Version(), obj2.Version(), obj4.Version()},
|
|
||||||
delList: []string{obj2.Version()},
|
|
||||||
isSorted: true,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
versions: &objectVersions{objects: []*data.ObjectInfo{obj5}},
|
|
||||||
objectToAdd: obj6,
|
|
||||||
expectedVersions: &objectVersions{
|
|
||||||
objects: []*data.ObjectInfo{obj5, obj6},
|
|
||||||
addList: []string{obj1.Version(), obj2.Version(), obj3.Version(), obj5.Version(), obj6.Version()},
|
|
||||||
isSorted: true,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
tc.versions.appendVersion(tc.objectToAdd)
|
|
||||||
tc.versions.sort()
|
|
||||||
require.Equal(t, tc.expectedVersions, tc.versions)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSortAddHeaders(t *testing.T) {
|
|
||||||
obj1 := getTestObjectInfo(1, "", "", "")
|
|
||||||
obj2 := getTestObjectInfo(2, "", "", "")
|
|
||||||
obj3 := getTestObjectInfo(3, "", "", "")
|
|
||||||
obj4 := getTestObjectInfo(4, "", "", "")
|
|
||||||
obj5 := getTestObjectInfo(5, "", "", "")
|
|
||||||
|
|
||||||
obj6 := getTestObjectInfoEpoch(1, 6, joinVers(obj1, obj2, obj3), "", "")
|
|
||||||
obj7 := getTestObjectInfoEpoch(1, 7, joinVers(obj1, obj4), "", "")
|
|
||||||
obj8 := getTestObjectInfoEpoch(1, 8, joinVers(obj5), "", "")
|
|
||||||
obj9 := getTestObjectInfoEpoch(1, 8, joinVers(obj1, obj5), "", "")
|
|
||||||
obj10 := getTestObjectInfo(11, "", "", "")
|
|
||||||
obj11 := getTestObjectInfo(10, joinVers(obj10), "", "")
|
|
||||||
obj12 := getTestObjectInfo(9, joinVers(obj10, obj11), "", "")
|
|
||||||
|
|
||||||
for _, tc := range []struct {
|
|
||||||
versions *objectVersions
|
|
||||||
expectedAddHeaders string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
versions: &objectVersions{objects: []*data.ObjectInfo{obj6, obj7, obj8}},
|
|
||||||
expectedAddHeaders: joinVers(obj1, obj2, obj3, obj4, obj5, obj6, obj7, obj8),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
versions: &objectVersions{objects: []*data.ObjectInfo{obj7, obj9}},
|
|
||||||
expectedAddHeaders: joinVers(obj1, obj4, obj5, obj7, obj9),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
versions: &objectVersions{objects: []*data.ObjectInfo{obj11, obj10, obj12}},
|
|
||||||
expectedAddHeaders: joinVers(obj10, obj11, obj12),
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
require.Equal(t, tc.expectedAddHeaders, tc.versions.getAddHeader())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func joinVers(objs ...*data.ObjectInfo) string {
|
|
||||||
if len(objs) == 0 {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
|
|
||||||
var versions []string
|
|
||||||
for _, obj := range objs {
|
|
||||||
versions = append(versions, obj.Version())
|
|
||||||
}
|
|
||||||
|
|
||||||
return strings.Join(versions, ",")
|
|
||||||
}
|
|
||||||
|
|
||||||
func getOID(id byte) oid.ID {
|
|
||||||
b := [32]byte{}
|
|
||||||
b[31] = id
|
|
||||||
|
|
||||||
var idObj oid.ID
|
|
||||||
idObj.SetSHA256(b)
|
|
||||||
return idObj
|
|
||||||
}
|
|
||||||
|
|
||||||
func getTestObjectInfo(id byte, addAttr, delAttr, delMarkAttr string) *data.ObjectInfo {
|
|
||||||
headers := make(map[string]string)
|
|
||||||
if addAttr != "" {
|
|
||||||
headers[versionsAddAttr] = addAttr
|
|
||||||
}
|
|
||||||
if delAttr != "" {
|
|
||||||
headers[versionsDelAttr] = delAttr
|
|
||||||
}
|
|
||||||
if delMarkAttr != "" {
|
|
||||||
headers[VersionsDeleteMarkAttr] = delMarkAttr
|
|
||||||
}
|
|
||||||
|
|
||||||
return &data.ObjectInfo{
|
|
||||||
ID: getOID(id),
|
|
||||||
Name: strconv.Itoa(int(id)),
|
|
||||||
Headers: headers,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func getTestObjectInfoEpoch(epoch uint64, id byte, addAttr, delAttr, delMarkAttr string) *data.ObjectInfo {
|
|
||||||
obj := getTestObjectInfo(id, addAttr, delAttr, delMarkAttr)
|
|
||||||
obj.CreationEpoch = epoch
|
|
||||||
return obj
|
|
||||||
}
|
|
||||||
|
|
Loading…
Reference in a new issue