From c50a16a5e3fdb8fd8054e97d92a9a57f7cd4a953 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Mon, 9 Aug 2021 17:29:44 +0300 Subject: [PATCH 01/10] [#122] Add enabling versioning New handlers: PutBucketVersioning, GetBucketVersioning Signed-off-by: Denis Kirillov --- api/handler/list.go | 21 -------- api/handler/not_support.go | 4 -- api/handler/response.go | 7 +++ api/handler/versioning.go | 64 +++++++++++++++++++++++ api/layer/layer.go | 102 +++++++++++++++++++++++++++++++++---- 5 files changed, 163 insertions(+), 35 deletions(-) create mode 100644 api/handler/versioning.go diff --git a/api/handler/list.go b/api/handler/list.go index f226d5dd..3fe83a0a 100644 --- a/api/handler/list.go +++ b/api/handler/list.go @@ -9,13 +9,6 @@ import ( "github.com/nspcc-dev/neofs-s3-gw/api" ) -// VersioningConfiguration contains VersioningConfiguration XML representation. -type VersioningConfiguration struct { - XMLName xml.Name `xml:"VersioningConfiguration"` - Text string `xml:",chardata"` - Xmlns string `xml:"xmlns,attr"` -} - // ListMultipartUploadsResult contains ListMultipartUploadsResult XML representation. type ListMultipartUploadsResult struct { XMLName xml.Name `xml:"ListMultipartUploadsResult"` @@ -62,20 +55,6 @@ func (h *handler) ListBucketsHandler(w http.ResponseWriter, r *http.Request) { } } -// GetBucketVersioningHandler implements bucket versioning getter handler. -func (h *handler) GetBucketVersioningHandler(w http.ResponseWriter, r *http.Request) { - var ( - reqInfo = api.GetReqInfo(r.Context()) - res = new(VersioningConfiguration) - ) - - res.Xmlns = "http://s3.amazonaws.com/doc/2006-03-01/" - - if err := api.EncodeToResponse(w, res); err != nil { - h.logAndSendError(w, "something went wrong", reqInfo, err) - } -} - // ListMultipartUploadsHandler implements multipart uploads listing handler. func (h *handler) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) { var ( diff --git a/api/handler/not_support.go b/api/handler/not_support.go index 6556eaf6..019fe5b4 100644 --- a/api/handler/not_support.go +++ b/api/handler/not_support.go @@ -23,10 +23,6 @@ func (h *handler) PutBucketTaggingHandler(w http.ResponseWriter, r *http.Request h.logAndSendError(w, "not supported", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotSupported)) } -func (h *handler) PutBucketVersioningHandler(w http.ResponseWriter, r *http.Request) { - h.logAndSendError(w, "not supported", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotSupported)) -} - func (h *handler) PutBucketNotificationHandler(w http.ResponseWriter, r *http.Request) { h.logAndSendError(w, "not supported", api.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotSupported)) } diff --git a/api/handler/response.go b/api/handler/response.go index 90f1fdab..7d344891 100644 --- a/api/handler/response.go +++ b/api/handler/response.go @@ -164,6 +164,13 @@ type ListObjectsVersionsResponse struct { CommonPrefixes []CommonPrefix `xml:"CommonPrefixes"` } +// VersioningConfiguration contains VersioningConfiguration XML representation. +type VersioningConfiguration struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ VersioningConfiguration"` + Status string `xml:"Status"` + MfaDelete string `xml:"MfaDelete,omitempty"` +} + // MarshalXML - StringMap marshals into XML. func (s StringMap) MarshalXML(e *xml.Encoder, start xml.StartElement) error { tokens := []xml.Token{start} diff --git a/api/handler/versioning.go b/api/handler/versioning.go new file mode 100644 index 00000000..3fc70680 --- /dev/null +++ b/api/handler/versioning.go @@ -0,0 +1,64 @@ +package handler + +import ( + "encoding/xml" + "net/http" + "strconv" + + "github.com/nspcc-dev/neofs-s3-gw/api" + "github.com/nspcc-dev/neofs-s3-gw/api/layer" + "go.uber.org/zap" +) + +func (h *handler) PutBucketVersioningHandler(w http.ResponseWriter, r *http.Request) { + reqInfo := api.GetReqInfo(r.Context()) + + configuration := new(VersioningConfiguration) + if err := xml.NewDecoder(r.Body).Decode(configuration); err != nil { + h.logAndSendError(w, "couldn't decode versioning configuration", reqInfo, api.GetAPIError(api.ErrIllegalVersioningConfigurationException)) + return + } + + p := &layer.PutVersioningParams{ + Bucket: reqInfo.BucketName, + VersioningEnabled: configuration.Status == "Enabled", + } + + if _, err := h.obj.PutBucketVersioning(r.Context(), p); err != nil { + h.logAndSendError(w, "couldn't put update versioning settings", reqInfo, err) + } +} + +// GetBucketVersioningHandler implements bucket versioning getter handler. +func (h *handler) GetBucketVersioningHandler(w http.ResponseWriter, r *http.Request) { + reqInfo := api.GetReqInfo(r.Context()) + + objInfo, err := h.obj.GetBucketVersioning(r.Context(), reqInfo.BucketName) + if err != nil { + h.log.Warn("couldn't get version settings object: default version settings will be used", + zap.String("request_id", reqInfo.RequestID), + zap.String("method", reqInfo.API), + zap.String("object_name", reqInfo.ObjectName), + zap.Error(err)) + } + + if err = api.EncodeToResponse(w, formVersioningConfiguration(objInfo)); err != nil { + h.logAndSendError(w, "something went wrong", reqInfo, err) + } +} + +func formVersioningConfiguration(inf *layer.ObjectInfo) *VersioningConfiguration { + res := &VersioningConfiguration{Status: "Suspended"} + + if inf == nil { + return res + } + + enabled, ok := inf.Headers["S3-Settings-Versioning-enabled"] + if ok { + if parsed, err := strconv.ParseBool(enabled); err == nil && parsed { + res.Status = "Enabled" + } + } + return res +} diff --git a/api/layer/layer.go b/api/layer/layer.go index fd7d0581..2ae3cc58 100644 --- a/api/layer/layer.go +++ b/api/layer/layer.go @@ -7,6 +7,7 @@ import ( "io" "net/url" "sort" + "strconv" "time" "github.com/nspcc-dev/neofs-api-go/pkg/acl/eacl" @@ -71,6 +72,12 @@ type ( Header map[string]string } + // PutVersioningParams stores object copy request parameters. + PutVersioningParams struct { + Bucket string + VersioningEnabled bool + } + // CopyObjectParams stores object copy request parameters. CopyObjectParams struct { SrcBucket string @@ -117,6 +124,9 @@ type ( Client interface { NeoFS + PutBucketVersioning(ctx context.Context, p *PutVersioningParams) (*ObjectInfo, error) + GetBucketVersioning(ctx context.Context, name string) (*ObjectInfo, error) + ListBuckets(ctx context.Context) ([]*BucketInfo, error) GetBucketInfo(ctx context.Context, name string) (*BucketInfo, error) GetBucketACL(ctx context.Context, name string) (*BucketACL, error) @@ -142,6 +152,7 @@ type ( const ( unversionedObjectVersionID = "null" + bktVersionSettingsObject = ".s3-versioning-settings" ) // NewLayer creates instance of layer. It checks credentials @@ -303,17 +314,18 @@ func (n *layer) checkObject(ctx context.Context, cid *cid.ID, filename string) e // GetObjectInfo returns meta information about the object. func (n *layer) GetObjectInfo(ctx context.Context, bucketName, filename string) (*ObjectInfo, error) { - var ( - err error - oid *object.ID - bkt *BucketInfo - meta *object.Object - ) - - if bkt, err = n.GetBucketInfo(ctx, bucketName); err != nil { + bkt, err := n.GetBucketInfo(ctx, bucketName) + if err != nil { n.log.Error("could not fetch bucket info", zap.Error(err)) return nil, err - } else if oid, err = n.objectFindID(ctx, &findParams{cid: bkt.CID, val: filename}); err != nil { + } + + return n.getObjectInfo(ctx, bkt, filename) +} + +func (n *layer) getObjectInfo(ctx context.Context, bkt *BucketInfo, objectName string) (*ObjectInfo, error) { + oid, err := n.objectFindID(ctx, &findParams{cid: bkt.CID, val: objectName}) + if err != nil { n.log.Error("could not find object id", zap.Error(err)) return nil, err } @@ -325,7 +337,7 @@ func (n *layer) GetObjectInfo(ctx context.Context, bucketName, filename string) /* todo: now we get an address via request to NeoFS and try to find the object with the address in cache but it will be resolved after implementation of local cache with nicenames and address of objects for get/head requests */ - meta = n.objCache.Get(addr) + meta := n.objCache.Get(addr) if meta == nil { meta, err = n.objectHead(ctx, addr) if err != nil { @@ -508,3 +520,73 @@ func (n *layer) ListObjectVersions(ctx context.Context, p *ListObjectVersionsPar } return &res, nil } + +func (n *layer) PutBucketVersioning(ctx context.Context, p *PutVersioningParams) (*ObjectInfo, error) { + bucketInfo, err := n.GetBucketInfo(ctx, p.Bucket) + if err != nil { + return nil, err + } + + objectInfo, err := n.getObjectInfo(ctx, bucketInfo, bktVersionSettingsObject) + if err != nil { + n.log.Warn("couldn't get bucket version settings object, new one will be created", + zap.String("bucket_name", bucketInfo.Name), + zap.Stringer("cid", bucketInfo.CID), + zap.String("object_name", bktVersionSettingsObject), + zap.Error(err)) + } + + attributes := make([]*object.Attribute, 0, 3) + + filename := object.NewAttribute() + filename.SetKey(object.AttributeFileName) + filename.SetValue(bktVersionSettingsObject) + + createdAt := object.NewAttribute() + createdAt.SetKey(object.AttributeTimestamp) + createdAt.SetValue(strconv.FormatInt(time.Now().UTC().Unix(), 10)) + + versioningIgnore := object.NewAttribute() + versioningIgnore.SetKey("S3-Versions-ignore") + versioningIgnore.SetValue(strconv.FormatBool(true)) + + settingsVersioningEnabled := object.NewAttribute() + settingsVersioningEnabled.SetKey("S3-Settings-Versioning-enabled") + settingsVersioningEnabled.SetValue(strconv.FormatBool(p.VersioningEnabled)) + + attributes = append(attributes, filename, createdAt, versioningIgnore, settingsVersioningEnabled) + + raw := object.NewRaw() + raw.SetOwnerID(bucketInfo.Owner) + raw.SetContainerID(bucketInfo.CID) + raw.SetAttributes(attributes...) + + ops := new(client.PutObjectParams).WithObject(raw.Object()) + oid, err := n.pool.PutObject(ctx, ops, n.BearerOpt(ctx)) + if err != nil { + return nil, err + } + + addr := object.NewAddress() + addr.SetObjectID(oid) + addr.SetContainerID(bucketInfo.CID) + meta, err := n.objectHead(ctx, addr) + if err != nil { + return nil, err + } + + if objectInfo != nil { + addr := object.NewAddress() + addr.SetObjectID(objectInfo.ID()) + addr.SetContainerID(bucketInfo.CID) + if err = n.objectDelete(ctx, addr); err != nil { + return nil, err + } + } + + return objectInfoFromMeta(bucketInfo, meta, "", ""), nil +} + +func (n *layer) GetBucketVersioning(ctx context.Context, bucketName string) (*ObjectInfo, error) { + return n.GetObjectInfo(ctx, bucketName, bktVersionSettingsObject) +} From feb45d0633a35a3cb6bcdd76325ac4ede0378a16 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Tue, 10 Aug 2021 11:19:09 +0300 Subject: [PATCH 02/10] [#122] Add replacing objects Signed-off-by: Denis Kirillov --- api/layer/layer.go | 15 ++++--- api/layer/object.go | 104 ++++++++++++++++++++++++++++++++++++++------ 2 files changed, 100 insertions(+), 19 deletions(-) diff --git a/api/layer/layer.go b/api/layer/layer.go index 2ae3cc58..e7d76af9 100644 --- a/api/layer/layer.go +++ b/api/layer/layer.go @@ -320,11 +320,11 @@ func (n *layer) GetObjectInfo(ctx context.Context, bucketName, filename string) return nil, err } - return n.getObjectInfo(ctx, bkt, filename) + return n.headLastVersion(ctx, bkt, filename) } -func (n *layer) getObjectInfo(ctx context.Context, bkt *BucketInfo, objectName string) (*ObjectInfo, error) { - oid, err := n.objectFindID(ctx, &findParams{cid: bkt.CID, val: objectName}) +func (n *layer) getSettingsObjectInfo(ctx context.Context, bkt *BucketInfo) (*ObjectInfo, error) { + oid, err := n.objectFindID(ctx, &findParams{cid: bkt.CID, val: bktVersionSettingsObject}) if err != nil { n.log.Error("could not find object id", zap.Error(err)) return nil, err @@ -527,12 +527,11 @@ func (n *layer) PutBucketVersioning(ctx context.Context, p *PutVersioningParams) return nil, err } - objectInfo, err := n.getObjectInfo(ctx, bucketInfo, bktVersionSettingsObject) + objectInfo, err := n.getSettingsObjectInfo(ctx, bucketInfo) if err != nil { n.log.Warn("couldn't get bucket version settings object, new one will be created", zap.String("bucket_name", bucketInfo.Name), zap.Stringer("cid", bucketInfo.CID), - zap.String("object_name", bktVersionSettingsObject), zap.Error(err)) } @@ -588,5 +587,9 @@ func (n *layer) PutBucketVersioning(ctx context.Context, p *PutVersioningParams) } func (n *layer) GetBucketVersioning(ctx context.Context, bucketName string) (*ObjectInfo, error) { - return n.GetObjectInfo(ctx, bucketName, bktVersionSettingsObject) + bktInfo, err := n.GetBucketInfo(ctx, bucketName) + if err != nil { + return nil, err + } + return n.getSettingsObjectInfo(ctx, bktInfo) } diff --git a/api/layer/object.go b/api/layer/object.go index 77d310d1..7ec3d1cf 100644 --- a/api/layer/object.go +++ b/api/layer/object.go @@ -3,6 +3,7 @@ package layer import ( "context" "errors" + "fmt" "io" "net/url" "sort" @@ -119,24 +120,37 @@ func (n *layer) objectPut(ctx context.Context, p *PutObjectParams) (*ObjectInfo, own = n.Owner(ctx) ) + if p.Object == bktVersionSettingsObject { + return nil, fmt.Errorf("trying put bucket settings object") + } + if obj, err = url.QueryUnescape(p.Object); err != nil { return nil, err - } else if bkt, err = n.GetBucketInfo(ctx, p.Bucket); err != nil { + } + if bkt, err = n.GetBucketInfo(ctx, p.Bucket); err != nil { return nil, err - } else if err = n.checkObject(ctx, bkt.CID, p.Object); err != nil { - var errExist *apiErrors.ObjectAlreadyExists - if ok := errors.As(err, &errExist); ok { - errExist.Bucket = p.Bucket - errExist.Object = p.Object - return nil, errExist - } + } - if !apiErrors.IsS3Error(err, apiErrors.ErrNoSuchKey) { - return nil, err - } + lastVersionInfo, err := n.headLastVersion(ctx, bkt, p.Object) + if err != nil && !apiErrors.IsS3Error(err, apiErrors.ErrNoSuchKey) { + return nil, err } attributes := make([]*object.Attribute, 0, len(p.Header)+1) + var idsToDeleteArr []*object.ID + if lastVersionInfo != nil { + versionsDeletedStr := lastVersionInfo.Headers["S3-Versions-del"] + if len(versionsDeletedStr) != 0 { + versionsDeletedStr += "," + } + versionsDeletedStr += lastVersionInfo.ID().String() + deletedVersions := object.NewAttribute() + deletedVersions.SetKey("S3-Versions-del") + deletedVersions.SetValue(versionsDeletedStr) + + attributes = append(attributes, deletedVersions) + idsToDeleteArr = append(idsToDeleteArr, lastVersionInfo.ID()) + } unix := strconv.FormatInt(time.Now().UTC().Unix(), 10) @@ -187,7 +201,7 @@ func (n *layer) objectPut(ctx context.Context, p *PutObjectParams) (*ObjectInfo, n.log.Error("couldn't cache an object", zap.Error(err)) } - return &ObjectInfo{ + objInfo := &ObjectInfo{ id: oid, Owner: own, @@ -198,7 +212,71 @@ func (n *layer) objectPut(ctx context.Context, p *PutObjectParams) (*ObjectInfo, Headers: p.Header, ContentType: r.contentType, HashSum: meta.PayloadChecksum().String(), - }, nil + } + + for _, id := range idsToDeleteArr { + addr := object.NewAddress() + addr.SetObjectID(id) + addr.SetContainerID(bkt.CID) + + if err = n.objectDelete(ctx, addr); err != nil { + n.log.Warn("couldn't delete object", + zap.Stringer("version id", id), + zap.Error(err)) + } + } + + return objInfo, nil +} + +func (n *layer) headLastVersion(ctx context.Context, bkt *BucketInfo, objectName string) (*ObjectInfo, error) { + ids, err := n.objectSearch(ctx, &findParams{cid: bkt.CID, val: objectName}) + if err != nil { + return nil, err + } + + if len(ids) == 0 { + return nil, api.GetAPIError(api.ErrNoSuchKey) + } + + infos := make([]*object.Object, 0, len(ids)) + for _, id := range ids { + addr := object.NewAddress() + addr.SetContainerID(bkt.CID) + addr.SetObjectID(id) + meta, err := n.objectHead(ctx, addr) + if err != nil { + n.log.Warn("couldn't head object", + zap.Stringer("object id", id), + zap.Stringer("bucket id", bkt.CID), + zap.Error(err)) + continue + } + infos = append(infos, meta) + } + + sort.Slice(infos, func(i, j int) bool { + return infos[i].CreationEpoch() < infos[j].CreationEpoch() || (infos[i].CreationEpoch() == infos[j].CreationEpoch() && infos[i].ID().String() < infos[j].ID().String()) + }) + + return objectInfoFromMeta(bkt, infos[len(infos)-1], "", ""), nil + //versionsToDeleteStr, ok := lastVersionInfo.Headers["S3-Versions-add"] + //versionsDeletedStr := lastVersionInfo.Headers["S3-Versions-del"] + //idsToDeleteArr := []*object.ID{lastVersionInfo.ID()} + //if ok { + // // for versioning mode only + // idsToDelete := strings.Split(versionsToDeleteStr, ",") + // for _, idStr := range idsToDelete { + // oid := object.NewID() + // if err = oid.Parse(idStr); err != nil { + // n.log.Warn("couldn't parse object id versions list", + // zap.String("versions id", versionsToDeleteStr), + // zap.Error(err)) + // break + // } + // idsToDeleteArr = append(idsToDeleteArr, oid) + // } + //} } // objectDelete puts tombstone object into neofs. From f463522f34d7b863a9a96482a8792953dc55dc6d Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Tue, 10 Aug 2021 11:58:40 +0300 Subject: [PATCH 03/10] [#122] Add versioning put object Signed-off-by: Denis Kirillov --- api/handler/versioning.go | 22 ++++++--------- api/layer/layer.go | 41 +++++++++++++++++++++++---- api/layer/object.go | 58 +++++++++++++++++++++++++++++++-------- 3 files changed, 90 insertions(+), 31 deletions(-) diff --git a/api/handler/versioning.go b/api/handler/versioning.go index 3fc70680..b69dfb43 100644 --- a/api/handler/versioning.go +++ b/api/handler/versioning.go @@ -3,7 +3,6 @@ package handler import ( "encoding/xml" "net/http" - "strconv" "github.com/nspcc-dev/neofs-s3-gw/api" "github.com/nspcc-dev/neofs-s3-gw/api/layer" @@ -20,8 +19,8 @@ func (h *handler) PutBucketVersioningHandler(w http.ResponseWriter, r *http.Requ } p := &layer.PutVersioningParams{ - Bucket: reqInfo.BucketName, - VersioningEnabled: configuration.Status == "Enabled", + Bucket: reqInfo.BucketName, + Settings: &layer.BucketSettings{VersioningEnabled: configuration.Status == "Enabled"}, } if _, err := h.obj.PutBucketVersioning(r.Context(), p); err != nil { @@ -33,7 +32,7 @@ func (h *handler) PutBucketVersioningHandler(w http.ResponseWriter, r *http.Requ func (h *handler) GetBucketVersioningHandler(w http.ResponseWriter, r *http.Request) { reqInfo := api.GetReqInfo(r.Context()) - objInfo, err := h.obj.GetBucketVersioning(r.Context(), reqInfo.BucketName) + settings, err := h.obj.GetBucketVersioning(r.Context(), reqInfo.BucketName) if err != nil { h.log.Warn("couldn't get version settings object: default version settings will be used", zap.String("request_id", reqInfo.RequestID), @@ -42,23 +41,18 @@ func (h *handler) GetBucketVersioningHandler(w http.ResponseWriter, r *http.Requ zap.Error(err)) } - if err = api.EncodeToResponse(w, formVersioningConfiguration(objInfo)); err != nil { + if err = api.EncodeToResponse(w, formVersioningConfiguration(settings)); err != nil { h.logAndSendError(w, "something went wrong", reqInfo, err) } } -func formVersioningConfiguration(inf *layer.ObjectInfo) *VersioningConfiguration { +func formVersioningConfiguration(settings *layer.BucketSettings) *VersioningConfiguration { res := &VersioningConfiguration{Status: "Suspended"} - - if inf == nil { + if settings == nil { return res } - - enabled, ok := inf.Headers["S3-Settings-Versioning-enabled"] - if ok { - if parsed, err := strconv.ParseBool(enabled); err == nil && parsed { - res.Status = "Enabled" - } + if settings.VersioningEnabled { + res.Status = "Enabled" } return res } diff --git a/api/layer/layer.go b/api/layer/layer.go index e7d76af9..420c4088 100644 --- a/api/layer/layer.go +++ b/api/layer/layer.go @@ -74,7 +74,12 @@ type ( // PutVersioningParams stores object copy request parameters. PutVersioningParams struct { - Bucket string + Bucket string + Settings *BucketSettings + } + + // BucketSettings stores settings such as versioning. + BucketSettings struct { VersioningEnabled bool } @@ -125,7 +130,7 @@ type ( NeoFS PutBucketVersioning(ctx context.Context, p *PutVersioningParams) (*ObjectInfo, error) - GetBucketVersioning(ctx context.Context, name string) (*ObjectInfo, error) + GetBucketVersioning(ctx context.Context, name string) (*BucketSettings, error) ListBuckets(ctx context.Context) ([]*BucketInfo, error) GetBucketInfo(ctx context.Context, name string) (*BucketInfo, error) @@ -551,7 +556,7 @@ func (n *layer) PutBucketVersioning(ctx context.Context, p *PutVersioningParams) settingsVersioningEnabled := object.NewAttribute() settingsVersioningEnabled.SetKey("S3-Settings-Versioning-enabled") - settingsVersioningEnabled.SetValue(strconv.FormatBool(p.VersioningEnabled)) + settingsVersioningEnabled.SetValue(strconv.FormatBool(p.Settings.VersioningEnabled)) attributes = append(attributes, filename, createdAt, versioningIgnore, settingsVersioningEnabled) @@ -586,10 +591,36 @@ func (n *layer) PutBucketVersioning(ctx context.Context, p *PutVersioningParams) return objectInfoFromMeta(bucketInfo, meta, "", ""), nil } -func (n *layer) GetBucketVersioning(ctx context.Context, bucketName string) (*ObjectInfo, error) { +func (n *layer) GetBucketVersioning(ctx context.Context, bucketName string) (*BucketSettings, error) { bktInfo, err := n.GetBucketInfo(ctx, bucketName) if err != nil { return nil, err } - return n.getSettingsObjectInfo(ctx, bktInfo) + + return n.getBucketSettings(ctx, bktInfo) +} + +func (n *layer) getBucketSettings(ctx context.Context, bktInfo *BucketInfo) (*BucketSettings, error) { + objInfo, err := n.getSettingsObjectInfo(ctx, bktInfo) + if err != nil { + return nil, err + } + + return objectInfoToBucketSettings(objInfo), nil +} + +func objectInfoToBucketSettings(info *ObjectInfo) *BucketSettings { + res := &BucketSettings{} + + if info == nil { + return res + } + + enabled, ok := info.Headers["S3-Settings-Versioning-enabled"] + if ok { + if parsed, err := strconv.ParseBool(enabled); err == nil { + res.VersioningEnabled = parsed + } + } + return res } diff --git a/api/layer/object.go b/api/layer/object.go index 7ec3d1cf..bd816988 100644 --- a/api/layer/object.go +++ b/api/layer/object.go @@ -62,6 +62,11 @@ type ( } ) +const ( + versionsDelAttr = "S3-Versions-del" + versionsAddAttr = "S3-Versions-add" +) + // objectSearch returns all available objects by search params. func (n *layer) objectSearch(ctx context.Context, p *findParams) ([]*object.ID, error) { var opts object.SearchFilters @@ -131,6 +136,7 @@ func (n *layer) objectPut(ctx context.Context, p *PutObjectParams) (*ObjectInfo, return nil, err } + versioningEnabled := n.isVersioningEnabled(ctx, bkt) lastVersionInfo, err := n.headLastVersion(ctx, bkt, p.Object) if err != nil && !apiErrors.IsS3Error(err, apiErrors.ErrNoSuchKey) { return nil, err @@ -139,17 +145,35 @@ func (n *layer) objectPut(ctx context.Context, p *PutObjectParams) (*ObjectInfo, attributes := make([]*object.Attribute, 0, len(p.Header)+1) var idsToDeleteArr []*object.ID if lastVersionInfo != nil { - versionsDeletedStr := lastVersionInfo.Headers["S3-Versions-del"] - if len(versionsDeletedStr) != 0 { - versionsDeletedStr += "," - } - versionsDeletedStr += lastVersionInfo.ID().String() - deletedVersions := object.NewAttribute() - deletedVersions.SetKey("S3-Versions-del") - deletedVersions.SetValue(versionsDeletedStr) + if versioningEnabled { + versionsAddedStr := lastVersionInfo.Headers[versionsAddAttr] + if len(versionsAddedStr) != 0 { + versionsAddedStr += "," + } + versionsAddedStr += lastVersionInfo.ID().String() + addedVersions := object.NewAttribute() + addedVersions.SetKey(versionsAddAttr) + addedVersions.SetValue(versionsAddedStr) + attributes = append(attributes, addedVersions) + if delVersions := lastVersionInfo.Headers[versionsDelAttr]; len(delVersions) > 0 { + deletedVersions := object.NewAttribute() + deletedVersions.SetKey(versionsDelAttr) + deletedVersions.SetValue(delVersions) + attributes = append(attributes, deletedVersions) + } + } else { + versionsDeletedStr := lastVersionInfo.Headers[versionsDelAttr] + if len(versionsDeletedStr) != 0 { + versionsDeletedStr += "," + } + versionsDeletedStr += lastVersionInfo.ID().String() + deletedVersions := object.NewAttribute() + deletedVersions.SetKey(versionsDelAttr) + deletedVersions.SetValue(versionsDeletedStr) - attributes = append(attributes, deletedVersions) - idsToDeleteArr = append(idsToDeleteArr, lastVersionInfo.ID()) + attributes = append(attributes, deletedVersions) + idsToDeleteArr = append(idsToDeleteArr, lastVersionInfo.ID()) + } } unix := strconv.FormatInt(time.Now().UTC().Unix(), 10) @@ -260,8 +284,8 @@ func (n *layer) headLastVersion(ctx context.Context, bkt *BucketInfo, objectName }) return objectInfoFromMeta(bkt, infos[len(infos)-1], "", ""), nil - //versionsToDeleteStr, ok := lastVersionInfo.Headers["S3-Versions-add"] - //versionsDeletedStr := lastVersionInfo.Headers["S3-Versions-del"] + //versionsToDeleteStr, ok := lastVersionInfo.Headers[versionsAddAttr] + //versionsDeletedStr := lastVersionInfo.Headers[versionsDelAttr] //idsToDeleteArr := []*object.ID{lastVersionInfo.ID()} //if ok { // // for versioning mode only @@ -475,3 +499,13 @@ func (n *layer) listAllObjects(ctx context.Context, p ListObjectsParamsCommon) ( return allObjects, nil } + +func (n *layer) isVersioningEnabled(ctx context.Context, bktInfo *BucketInfo) bool { + settings, err := n.getBucketSettings(ctx, bktInfo) + if err != nil { + n.log.Warn("couldn't get versioning settings object", zap.Error(err)) + return false + } + + return settings.VersioningEnabled +} From 3130784ee6d4bb98dad10bbb45e2a7c360817ee2 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Tue, 10 Aug 2021 13:03:09 +0300 Subject: [PATCH 04/10] [#122] Add getting specific object version Signed-off-by: Denis Kirillov --- api/handler/copy.go | 10 +++-- api/handler/get.go | 17 +++++--- api/handler/head.go | 16 +++++--- api/handler/versioning.go | 1 + api/headers.go | 1 + api/layer/layer.go | 81 +++++++++++++++++++-------------------- api/layer/object.go | 64 ++++++++++++++++--------------- api/layer/util.go | 7 ++++ 8 files changed, 111 insertions(+), 86 deletions(-) diff --git a/api/handler/copy.go b/api/handler/copy.go index 5ebf36eb..0503a837 100644 --- a/api/handler/copy.go +++ b/api/handler/copy.go @@ -63,6 +63,11 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) { h.logAndSendError(w, "could not parse request params", reqInfo, err) return } + p := &layer.HeadObjectParams{ + Bucket: srcBucket, + Object: srcObject, + VersionID: reqInfo.URL.Query().Get("versionId"), + } if args.MetadataDirective == replaceMetadataDirective { metadata = parseMetadata(r) @@ -80,7 +85,7 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) { return } - if inf, err = h.obj.GetObjectInfo(r.Context(), srcBucket, srcObject); err != nil { + if inf, err = h.obj.GetObjectInfo(r.Context(), p); err != nil { h.logAndSendError(w, "could not find object", reqInfo, err) return } @@ -100,9 +105,8 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) { } params := &layer.CopyObjectParams{ - SrcBucket: srcBucket, + SrcObject: inf, DstBucket: reqInfo.BucketName, - SrcObject: srcObject, DstObject: reqInfo.ObjectName, SrcSize: inf.Size, Header: metadata, diff --git a/api/handler/get.go b/api/handler/get.go index 2233f28f..d18c1751 100644 --- a/api/handler/get.go +++ b/api/handler/get.go @@ -72,6 +72,7 @@ func writeHeaders(h http.Header, info *layer.ObjectInfo) { h.Set(api.LastModified, info.Created.UTC().Format(http.TimeFormat)) h.Set(api.ContentLength, strconv.FormatInt(info.Size, 10)) h.Set(api.ETag, info.HashSum) + h.Set(api.AmzVersionId, info.ID().String()) for key, val := range info.Headers { h[api.MetadataPrefix+key] = []string{val} @@ -98,7 +99,13 @@ func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) { return } - if inf, err = h.obj.GetObjectInfo(r.Context(), reqInfo.BucketName, reqInfo.ObjectName); err != nil { + p := &layer.HeadObjectParams{ + Bucket: reqInfo.BucketName, + Object: reqInfo.ObjectName, + VersionID: reqInfo.URL.Query().Get("versionId"), + } + + if inf, err = h.obj.GetObjectInfo(r.Context(), p); err != nil { h.logAndSendError(w, "could not find object", reqInfo, err) return } @@ -118,10 +125,10 @@ func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) { } getParams := &layer.GetObjectParams{ - Bucket: inf.Bucket, - Object: inf.Name, - Writer: w, - Range: params, + ObjectInfo: inf, + Writer: w, + Range: params, + VersionID: p.VersionID, } if err = h.obj.GetObject(r.Context(), getParams); err != nil { h.logAndSendError(w, "could not get object", reqInfo, err) diff --git a/api/handler/head.go b/api/handler/head.go index 128a2836..f6f1a274 100644 --- a/api/handler/head.go +++ b/api/handler/head.go @@ -36,16 +36,22 @@ func (h *handler) HeadObjectHandler(w http.ResponseWriter, r *http.Request) { return } - if inf, err = h.obj.GetObjectInfo(r.Context(), reqInfo.BucketName, reqInfo.ObjectName); err != nil { + p := &layer.HeadObjectParams{ + Bucket: reqInfo.BucketName, + Object: reqInfo.ObjectName, + VersionID: reqInfo.URL.Query().Get("versionId"), + } + + if inf, err = h.obj.GetObjectInfo(r.Context(), p); err != nil { h.logAndSendError(w, "could not fetch object info", reqInfo, err) return } buffer := bytes.NewBuffer(make([]byte, 0, sizeToDetectType)) getParams := &layer.GetObjectParams{ - Bucket: inf.Bucket, - Object: inf.Name, - Writer: buffer, - Range: getRangeToDetectContentType(inf.Size), + ObjectInfo: inf, + Writer: buffer, + Range: getRangeToDetectContentType(inf.Size), + VersionID: reqInfo.URL.Query().Get("versionId"), } if err = h.obj.GetObject(r.Context(), getParams); err != nil { h.logAndSendError(w, "could not get object", reqInfo, err, zap.Stringer("oid", inf.ID())) diff --git a/api/handler/versioning.go b/api/handler/versioning.go index b69dfb43..ece2e85c 100644 --- a/api/handler/versioning.go +++ b/api/handler/versioning.go @@ -39,6 +39,7 @@ func (h *handler) GetBucketVersioningHandler(w http.ResponseWriter, r *http.Requ zap.String("method", reqInfo.API), zap.String("object_name", reqInfo.ObjectName), zap.Error(err)) + return } if err = api.EncodeToResponse(w, formVersioningConfiguration(settings)); err != nil { diff --git a/api/headers.go b/api/headers.go index 0eaef4fc..303b46da 100644 --- a/api/headers.go +++ b/api/headers.go @@ -4,6 +4,7 @@ package api const ( MetadataPrefix = "X-Amz-Meta-" AmzMetadataDirective = "X-Amz-Metadata-Directive" + AmzVersionId = "X-Amz-Version-Id" LastModified = "Last-Modified" Date = "Date" diff --git a/api/layer/layer.go b/api/layer/layer.go index 420c4088..bbd6e0f7 100644 --- a/api/layer/layer.go +++ b/api/layer/layer.go @@ -49,12 +49,21 @@ type ( // GetObjectParams stores object get request parameters. GetObjectParams struct { - Range *RangeParams - Bucket string - Object string - Offset int64 - Length int64 - Writer io.Writer + Range *RangeParams + ObjectInfo *ObjectInfo + //Bucket string + //Object string + Offset int64 + Length int64 + Writer io.Writer + VersionID string + } + + // HeadObjectParams stores object head request parameters. + HeadObjectParams struct { + Bucket string + Object string + VersionID string } // RangeParams stores range header request parameters. @@ -85,9 +94,8 @@ type ( // CopyObjectParams stores object copy request parameters. CopyObjectParams struct { - SrcBucket string + SrcObject *ObjectInfo DstBucket string - SrcObject string DstObject string SrcSize int64 Header map[string]string @@ -140,7 +148,7 @@ type ( DeleteBucket(ctx context.Context, p *DeleteBucketParams) error GetObject(ctx context.Context, p *GetObjectParams) error - GetObjectInfo(ctx context.Context, bucketName, objectName string) (*ObjectInfo, error) + GetObjectInfo(ctx context.Context, p *HeadObjectParams) (*ObjectInfo, error) PutObject(ctx context.Context, p *PutObjectParams) (*ObjectInfo, error) @@ -265,21 +273,17 @@ func (n *layer) ListBuckets(ctx context.Context) ([]*BucketInfo, error) { // GetObject from storage. func (n *layer) GetObject(ctx context.Context, p *GetObjectParams) error { - var ( - err error - oid *object.ID - bkt *BucketInfo - ) + var err error - if bkt, err = n.GetBucketInfo(ctx, p.Bucket); err != nil { - return fmt.Errorf("couldn't find bucket: %s : %w", p.Bucket, err) - } else if oid, err = n.objectFindID(ctx, &findParams{cid: bkt.CID, val: p.Object}); err != nil { - return fmt.Errorf("search of the object failed: cid: %s, val: %s : %w", bkt.CID, p.Object, err) - } + //if bkt, err = n.GetBucketInfo(ctx, p.Bucket); err != nil { + // return fmt.Errorf("couldn't find bucket: %s : %w", p.Bucket, err) + //} else if oid, err = n.objectFindID(ctx, &findParams{cid: bkt.CID, val: p.Object}); err != nil { + // return fmt.Errorf("search of the object failed: cid: %s, val: %s : %w", bkt.CID, p.Object, err) + //} addr := object.NewAddress() - addr.SetObjectID(oid) - addr.SetContainerID(bkt.CID) + addr.SetObjectID(p.ObjectInfo.ID()) + addr.SetContainerID(p.ObjectInfo.CID()) params := &getParams{ Writer: p.Writer, @@ -301,7 +305,7 @@ func (n *layer) GetObject(ctx context.Context, p *GetObjectParams) error { if err != nil { n.objCache.Delete(addr) - return fmt.Errorf("couldn't get object, cid: %s : %w", bkt.CID, err) + return fmt.Errorf("couldn't get object, cid: %s : %w", p.ObjectInfo.CID(), err) } return nil @@ -318,14 +322,18 @@ func (n *layer) checkObject(ctx context.Context, cid *cid.ID, filename string) e } // GetObjectInfo returns meta information about the object. -func (n *layer) GetObjectInfo(ctx context.Context, bucketName, filename string) (*ObjectInfo, error) { - bkt, err := n.GetBucketInfo(ctx, bucketName) +func (n *layer) GetObjectInfo(ctx context.Context, p *HeadObjectParams) (*ObjectInfo, error) { + bkt, err := n.GetBucketInfo(ctx, p.Bucket) if err != nil { n.log.Error("could not fetch bucket info", zap.Error(err)) return nil, err } - return n.headLastVersion(ctx, bkt, filename) + if len(p.VersionID) == 0 { + return n.headLastVersion(ctx, bkt, p.Object) + } + + return n.headVersion(ctx, bkt, p.Object, p.VersionID) } func (n *layer) getSettingsObjectInfo(ctx context.Context, bkt *BucketInfo) (*ObjectInfo, error) { @@ -344,7 +352,7 @@ func (n *layer) getSettingsObjectInfo(ctx context.Context, bkt *BucketInfo) (*Ob for get/head requests */ meta := n.objCache.Get(addr) if meta == nil { - meta, err = n.objectHead(ctx, addr) + meta, err = n.objectHead(ctx, bkt.CID, oid) if err != nil { n.log.Error("could not fetch object head", zap.Error(err)) return nil, err @@ -353,6 +361,7 @@ func (n *layer) getSettingsObjectInfo(ctx context.Context, bkt *BucketInfo) (*Ob n.log.Error("couldn't cache an object", zap.Error(err)) } } + return objectInfoFromMeta(bkt, meta, "", ""), nil } @@ -367,9 +376,8 @@ func (n *layer) CopyObject(ctx context.Context, p *CopyObjectParams) (*ObjectInf go func() { err := n.GetObject(ctx, &GetObjectParams{ - Bucket: p.SrcBucket, - Object: p.SrcObject, - Writer: pw, + ObjectInfo: p.SrcObject, + Writer: pw, }) if err = pw.CloseWithError(err); err != nil { @@ -477,11 +485,7 @@ func (n *layer) ListObjectVersions(ctx context.Context, p *ListObjectVersionsPar res.DeleteMarker = deleted for _, id := range ids { - addr := object.NewAddress() - addr.SetObjectID(id) - addr.SetContainerID(bkt.CID) - - meta, err := n.objectHead(ctx, addr) + meta, err := n.objectHead(ctx, bkt.CID, id) if err != nil { n.log.Warn("could not fetch object meta", zap.Error(err)) continue @@ -571,10 +575,7 @@ func (n *layer) PutBucketVersioning(ctx context.Context, p *PutVersioningParams) return nil, err } - addr := object.NewAddress() - addr.SetObjectID(oid) - addr.SetContainerID(bucketInfo.CID) - meta, err := n.objectHead(ctx, addr) + meta, err := n.objectHead(ctx, bucketInfo.CID, oid) if err != nil { return nil, err } @@ -612,10 +613,6 @@ func (n *layer) getBucketSettings(ctx context.Context, bktInfo *BucketInfo) (*Bu func objectInfoToBucketSettings(info *ObjectInfo) *BucketSettings { res := &BucketSettings{} - if info == nil { - return res - } - enabled, ok := info.Headers["S3-Settings-Versioning-enabled"] if ok { if parsed, err := strconv.ParseBool(enabled); err == nil { diff --git a/api/layer/object.go b/api/layer/object.go index bd816988..44b2258d 100644 --- a/api/layer/object.go +++ b/api/layer/object.go @@ -8,6 +8,7 @@ import ( "net/url" "sort" "strconv" + "strings" "time" "github.com/nspcc-dev/neofs-api-go/pkg/client" @@ -96,7 +97,10 @@ func (n *layer) objectFindID(ctx context.Context, p *findParams) (*object.ID, er } // objectHead returns all object's headers. -func (n *layer) objectHead(ctx context.Context, address *object.Address) (*object.Object, error) { +func (n *layer) objectHead(ctx context.Context, cid *cid.ID, oid *object.ID) (*object.Object, error) { + address := object.NewAddress() + address.SetContainerID(cid) + address.SetObjectID(oid) ops := new(client.ObjectHeaderParams).WithAddress(address).WithAllFields() return n.pool.GetObjectHeader(ctx, ops, n.BearerOpt(ctx)) } @@ -213,10 +217,7 @@ func (n *layer) objectPut(ctx context.Context, p *PutObjectParams) (*ObjectInfo, return nil, err } - addr := object.NewAddress() - addr.SetObjectID(oid) - addr.SetContainerID(bkt.CID) - meta, err := n.objectHead(ctx, addr) + meta, err := n.objectHead(ctx, bkt.CID, oid) if err != nil { return nil, err } @@ -265,10 +266,7 @@ func (n *layer) headLastVersion(ctx context.Context, bkt *BucketInfo, objectName infos := make([]*object.Object, 0, len(ids)) for _, id := range ids { - addr := object.NewAddress() - addr.SetContainerID(bkt.CID) - addr.SetObjectID(id) - meta, err := n.objectHead(ctx, addr) + meta, err := n.objectHead(ctx, bkt.CID, id) if err != nil { n.log.Warn("couldn't head object", zap.Stringer("object id", id), @@ -284,23 +282,31 @@ func (n *layer) headLastVersion(ctx context.Context, bkt *BucketInfo, objectName }) return objectInfoFromMeta(bkt, infos[len(infos)-1], "", ""), nil - //versionsToDeleteStr, ok := lastVersionInfo.Headers[versionsAddAttr] - //versionsDeletedStr := lastVersionInfo.Headers[versionsDelAttr] - //idsToDeleteArr := []*object.ID{lastVersionInfo.ID()} - //if ok { - // // for versioning mode only - // idsToDelete := strings.Split(versionsToDeleteStr, ",") - // for _, idStr := range idsToDelete { - // oid := object.NewID() - // if err = oid.Parse(idStr); err != nil { - // n.log.Warn("couldn't parse object id versions list", - // zap.String("versions id", versionsToDeleteStr), - // zap.Error(err)) - // break - // } - // idsToDeleteArr = append(idsToDeleteArr, oid) - // } - //} +} + +func (n *layer) headVersion(ctx context.Context, bkt *BucketInfo, objectName, versionID string) (*ObjectInfo, error) { + ids, err := n.objectSearch(ctx, &findParams{cid: bkt.CID, val: objectName}) + if err != nil { + return nil, err + } + if len(ids) == 0 { + return nil, api.GetAPIError(api.ErrNoSuchVersion) + } + + for _, id := range ids { + if id.String() == versionID { + meta, err := n.objectHead(ctx, bkt.CID, id) + if err != nil { + if strings.Contains(err.Error(), "not found") { + return nil, api.GetAPIError(api.ErrNoSuchVersion) + } + return nil, err + } + return objectInfoFromMeta(bkt, meta, "", ""), nil + } + } + + return nil, api.GetAPIError(api.ErrNoSuchVersion) } // objectDelete puts tombstone object into neofs. @@ -399,11 +405,7 @@ func (n *layer) listSortedObjectsFromNeoFS(ctx context.Context, p allObjectParam objects := make([]*ObjectInfo, 0, len(ids)) for _, id := range ids { - addr := object.NewAddress() - addr.SetObjectID(id) - addr.SetContainerID(p.Bucket.CID) - - meta, err := n.objectHead(ctx, addr) + meta, err := n.objectHead(ctx, p.Bucket.CID, id) if err != nil { n.log.Warn("could not fetch object meta", zap.Error(err)) continue diff --git a/api/layer/util.go b/api/layer/util.go index 5bcee8b3..990c00ac 100644 --- a/api/layer/util.go +++ b/api/layer/util.go @@ -8,6 +8,8 @@ import ( "strings" "time" + cid "github.com/nspcc-dev/neofs-api-go/pkg/container/id" + "github.com/nspcc-dev/neofs-api-go/pkg/object" "github.com/nspcc-dev/neofs-api-go/pkg/owner" "github.com/nspcc-dev/neofs-s3-gw/api" @@ -21,6 +23,7 @@ type ( isDir bool Bucket string + bucketID *cid.ID Name string Size int64 ContentType string @@ -137,6 +140,7 @@ func objectInfoFromMeta(bkt *BucketInfo, meta *object.Object, prefix, delimiter isDir: isDir, Bucket: bkt.Name, + bucketID: bkt.CID, Name: filename, Created: creation, ContentType: mimeType, @@ -174,6 +178,9 @@ func NameFromString(name string) (string, string) { // ID returns object ID from ObjectInfo. func (o *ObjectInfo) ID() *object.ID { return o.id } +// CID returns bucket ID from ObjectInfo. +func (o *ObjectInfo) CID() *cid.ID { return o.bucketID } + // IsDir allows to check if object is a directory. func (o *ObjectInfo) IsDir() bool { return o.isDir } From d5aef7566f5d3b15ebb32f05d2ac9f320b021265 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Tue, 10 Aug 2021 15:08:15 +0300 Subject: [PATCH 05/10] [#122] Add delete versioned object Signed-off-by: Denis Kirillov --- api/errors/errors.go | 7 ++ api/handler/delete.go | 31 +++++++-- api/handler/get.go | 2 +- api/headers.go | 2 +- api/layer/layer.go | 154 ++++++++++++++++++++++++++++++++--------- api/layer/object.go | 78 +++++++++------------ api/layer/util_test.go | 1 + 7 files changed, 189 insertions(+), 86 deletions(-) diff --git a/api/errors/errors.go b/api/errors/errors.go index 24424a6b..d1dd1126 100644 --- a/api/errors/errors.go +++ b/api/errors/errors.go @@ -66,6 +66,7 @@ const ( ErrNoSuchKey ErrNoSuchUpload ErrNoSuchVersion + ErrInvalidVersion ErrNotImplemented ErrPreconditionFailed ErrNotModified @@ -529,6 +530,12 @@ var errorCodes = errorCodeMap{ Description: "Indicates that the version ID specified in the request does not match an existing version.", HTTPStatusCode: http.StatusNotFound, }, + ErrInvalidVersion: { + ErrCode: ErrInvalidVersion, + Code: "InvalidArgument", + Description: "Invalid version id specified", + HTTPStatusCode: http.StatusBadRequest, + }, ErrNotImplemented: { ErrCode: ErrNotImplemented, Code: "NotImplemented", diff --git a/api/handler/delete.go b/api/handler/delete.go index 742a7bbb..a8b56bc6 100644 --- a/api/handler/delete.go +++ b/api/handler/delete.go @@ -8,6 +8,7 @@ import ( "github.com/nspcc-dev/neofs-s3-gw/api/errors" "github.com/nspcc-dev/neofs-s3-gw/api/layer" "go.uber.org/zap" + "go.uber.org/zap/zapcore" ) // DeleteObjectsRequest - xml carrying the object key names which needs to be deleted. @@ -21,6 +22,7 @@ type DeleteObjectsRequest struct { // ObjectIdentifier carries key name for the object to delete. type ObjectIdentifier struct { ObjectName string `xml:"Key"` + VersionID string `xml:"VersionId,omitempty"` } // DeleteError structure. @@ -43,18 +45,22 @@ type DeleteObjectsResponse struct { func (h *handler) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) { reqInfo := api.GetReqInfo(r.Context()) + versionedObject := []*layer.VersionedObject{{ + Name: reqInfo.ObjectName, + VersionID: reqInfo.URL.Query().Get("versionId"), + }} if err := h.checkBucketOwner(r, reqInfo.BucketName); err != nil { h.logAndSendError(w, "expected owner doesn't match", reqInfo, err) return } - if err := h.obj.DeleteObject(r.Context(), reqInfo.BucketName, reqInfo.ObjectName); err != nil { + if errs := h.obj.DeleteObjects(r.Context(), reqInfo.BucketName, versionedObject); len(errs) != 0 && errs[0] != nil { h.log.Error("could not delete object", zap.String("request_id", reqInfo.RequestID), zap.String("bucket_name", reqInfo.BucketName), zap.String("object_name", reqInfo.ObjectName), - zap.Error(err)) + zap.Error(errs[0])) // Ignore delete errors: @@ -94,10 +100,14 @@ func (h *handler) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Re } removed := make(map[string]struct{}) - toRemove := make([]string, 0, len(requested.Objects)) + toRemove := make([]*layer.VersionedObject, 0, len(requested.Objects)) for _, obj := range requested.Objects { - removed[obj.ObjectName] = struct{}{} - toRemove = append(toRemove, obj.ObjectName) + versionedObj := &layer.VersionedObject{ + Name: obj.ObjectName, + VersionID: obj.VersionID, + } + toRemove = append(toRemove, versionedObj) + removed[versionedObj.String()] = struct{}{} } response := &DeleteObjectsResponse{ @@ -110,9 +120,16 @@ func (h *handler) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Re return } + marshaler := zapcore.ArrayMarshalerFunc(func(encoder zapcore.ArrayEncoder) error { + for _, obj := range toRemove { + encoder.AppendString(obj.String()) + } + return nil + }) + if errs := h.obj.DeleteObjects(r.Context(), reqInfo.BucketName, toRemove); errs != nil && !requested.Quiet { additional := []zap.Field{ - zap.Strings("objects_name", toRemove), + zap.Array("objects", marshaler), zap.Errors("errors", errs), } h.logAndSendError(w, "could not delete objects", reqInfo, nil, additional...) @@ -138,7 +155,7 @@ func (h *handler) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Re } if err := api.EncodeToResponse(w, response); err != nil { - h.logAndSendError(w, "could not write response", reqInfo, err, zap.Strings("objects_name", toRemove)) + h.logAndSendError(w, "could not write response", reqInfo, err, zap.Array("objects", marshaler)) return } } diff --git a/api/handler/get.go b/api/handler/get.go index d18c1751..9a1c62f0 100644 --- a/api/handler/get.go +++ b/api/handler/get.go @@ -72,7 +72,7 @@ func writeHeaders(h http.Header, info *layer.ObjectInfo) { h.Set(api.LastModified, info.Created.UTC().Format(http.TimeFormat)) h.Set(api.ContentLength, strconv.FormatInt(info.Size, 10)) h.Set(api.ETag, info.HashSum) - h.Set(api.AmzVersionId, info.ID().String()) + h.Set(api.AmzVersionID, info.ID().String()) for key, val := range info.Headers { h[api.MetadataPrefix+key] = []string{val} diff --git a/api/headers.go b/api/headers.go index 303b46da..adf81c5c 100644 --- a/api/headers.go +++ b/api/headers.go @@ -4,7 +4,7 @@ package api const ( MetadataPrefix = "X-Amz-Meta-" AmzMetadataDirective = "X-Amz-Metadata-Directive" - AmzVersionId = "X-Amz-Version-Id" + AmzVersionID = "X-Amz-Version-Id" LastModified = "Last-Modified" Date = "Date" diff --git a/api/layer/layer.go b/api/layer/layer.go index bbd6e0f7..a0a45a50 100644 --- a/api/layer/layer.go +++ b/api/layer/layer.go @@ -1,6 +1,7 @@ package layer import ( + "bytes" "context" "crypto/ecdsa" "fmt" @@ -8,6 +9,7 @@ import ( "net/url" "sort" "strconv" + "strings" "time" "github.com/nspcc-dev/neofs-api-go/pkg/acl/eacl" @@ -128,6 +130,12 @@ type ( Encode string } + // VersionedObject stores object name and version. + VersionedObject struct { + Name string + VersionID string + } + // NeoFS provides basic NeoFS interface. NeoFS interface { Get(ctx context.Context, address *object.Address) (*object.Object, error) @@ -158,8 +166,7 @@ type ( ListObjectsV2(ctx context.Context, p *ListObjectsParamsV2) (*ListObjectsInfoV2, error) ListObjectVersions(ctx context.Context, p *ListObjectVersionsParams) (*ListObjectVersionsInfo, error) - DeleteObject(ctx context.Context, bucket, object string) error - DeleteObjects(ctx context.Context, bucket string, objects []string) []error + DeleteObjects(ctx context.Context, bucket string, objects []*VersionedObject) []error } ) @@ -168,6 +175,10 @@ const ( bktVersionSettingsObject = ".s3-versioning-settings" ) +func (t *VersionedObject) String() string { + return t.Name + ":" + t.VersionID +} + // NewLayer creates instance of layer. It checks credentials // and establishes gRPC connection with node. func NewLayer(log *zap.Logger, conns pool.Pool, config *CacheConfig) Client { @@ -330,16 +341,21 @@ func (n *layer) GetObjectInfo(ctx context.Context, p *HeadObjectParams) (*Object } if len(p.VersionID) == 0 { - return n.headLastVersion(ctx, bkt, p.Object) + objInfo, err := n.headLastVersion(ctx, bkt, p.Object) + if err == nil { + if deleteMark, err2 := strconv.ParseBool(objInfo.Headers[versionsDeleteMarkAttr]); err2 == nil && deleteMark { + return nil, api.GetAPIError(api.ErrNoSuchKey) + } + } + return objInfo, err } - return n.headVersion(ctx, bkt, p.Object, p.VersionID) + return n.headVersion(ctx, bkt, p.VersionID) } func (n *layer) getSettingsObjectInfo(ctx context.Context, bkt *BucketInfo) (*ObjectInfo, error) { oid, err := n.objectFindID(ctx, &findParams{cid: bkt.CID, val: bktVersionSettingsObject}) if err != nil { - n.log.Error("could not find object id", zap.Error(err)) return nil, err } @@ -367,7 +383,12 @@ func (n *layer) getSettingsObjectInfo(ctx context.Context, bkt *BucketInfo) (*Ob // PutObject into storage. func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*ObjectInfo, error) { - return n.objectPut(ctx, p) + bkt, err := n.GetBucketInfo(ctx, p.Bucket) + if err != nil { + return nil, err + } + + return n.objectPut(ctx, bkt, p) } // CopyObject from one bucket into another bucket. @@ -395,35 +416,96 @@ func (n *layer) CopyObject(ctx context.Context, p *CopyObjectParams) (*ObjectInf } // DeleteObject removes all objects with passed nice name. -func (n *layer) DeleteObject(ctx context.Context, bucket, filename string) error { +func (n *layer) deleteObject(ctx context.Context, bkt *BucketInfo, obj *VersionedObject) error { var ( err error ids []*object.ID - bkt *BucketInfo ) - if bkt, err = n.GetBucketInfo(ctx, bucket); err != nil { - return &errors.DeleteError{ - Err: err, - Object: filename, + versioningEnabled := n.isVersioningEnabled(ctx, bkt) + if !versioningEnabled && obj.VersionID != "null" && obj.VersionID != "" { + return errors.GetAPIError(errors.ErrInvalidVersion) + } + + if versioningEnabled { + if len(obj.VersionID) != 0 { + id := object.NewID() + if err := id.Parse(obj.VersionID); err != nil { + return &errors.DeleteError{Err: api.GetAPIError(api.ErrInvalidVersion), Object: obj.String()} + } + ids = []*object.ID{id} + + lastObject, err := n.headLastVersion(ctx, bkt, obj.Name) + if err != nil { + return &api.DeleteError{Err: err, Object: obj.String()} + } + if !strings.Contains(lastObject.Headers[versionsAddAttr], obj.VersionID) || + strings.Contains(lastObject.Headers[versionsDelAttr], obj.VersionID) { + return &api.DeleteError{Err: api.GetAPIError(api.ErrInvalidVersion), Object: obj.String()} + } + + if lastObject.ID().String() == obj.VersionID { + if added := lastObject.Headers[versionsAddAttr]; len(added) > 0 { + addedVersions := strings.Split(added, ",") + sourceCopyVersion, err := n.headVersion(ctx, bkt, addedVersions[len(addedVersions)-1]) + if err != nil { + return &api.DeleteError{Err: err, Object: obj.String()} + } + p := &CopyObjectParams{ + SrcObject: sourceCopyVersion, + DstBucket: bkt.Name, + DstObject: obj.Name, + SrcSize: sourceCopyVersion.Size, + Header: map[string]string{versionsDelAttr: obj.VersionID}, + } + if _, err := n.CopyObject(ctx, p); err != nil { + return err + } + } else { + p := &PutObjectParams{ + Object: obj.Name, + Reader: bytes.NewReader(nil), + Header: map[string]string{ + versionsDelAttr: obj.VersionID, + versionsDeleteMarkAttr: strconv.FormatBool(true), + }, + } + if _, err := n.objectPut(ctx, bkt, p); err != nil { + return &api.DeleteError{Err: err, Object: obj.String()} + } + } + } else { + p := &CopyObjectParams{ + SrcObject: lastObject, + DstBucket: bkt.Name, + DstObject: obj.Name, + SrcSize: lastObject.Size, + Header: map[string]string{versionsDelAttr: obj.VersionID}, + } + if _, err := n.CopyObject(ctx, p); err != nil { + return err + } + } + } else { + p := &PutObjectParams{ + Object: obj.Name, + Reader: bytes.NewReader(nil), + Header: map[string]string{versionsDeleteMarkAttr: strconv.FormatBool(true)}, + } + if _, err := n.objectPut(ctx, bkt, p); err != nil { + return &errors.DeleteError{Err: err, Object: obj.String()} + } } - } else if ids, err = n.objectSearch(ctx, &findParams{cid: bkt.CID, val: filename}); err != nil { - return &errors.DeleteError{ - Err: err, - Object: filename, + } else { + ids, err = n.objectSearch(ctx, &findParams{cid: bkt.CID, val: obj.Name}) + if err != nil { + return &errors.DeleteError{Err: err, Object: obj.String()} } } for _, id := range ids { - addr := object.NewAddress() - addr.SetObjectID(id) - addr.SetContainerID(bkt.CID) - - if err = n.objectDelete(ctx, addr); err != nil { - return &errors.DeleteError{ - Err: err, - Object: filename, - } + if err = n.objectDelete(ctx, bkt.CID, id); err != nil { + return &errors.DeleteError{Err: err, Object: obj.String()} } } @@ -431,11 +513,16 @@ func (n *layer) DeleteObject(ctx context.Context, bucket, filename string) error } // DeleteObjects from the storage. -func (n *layer) DeleteObjects(ctx context.Context, bucket string, objects []string) []error { +func (n *layer) DeleteObjects(ctx context.Context, bucket string, objects []*VersionedObject) []error { var errs = make([]error, 0, len(objects)) + bkt, err := n.GetBucketInfo(ctx, bucket) + if err != nil { + return append(errs, err) + } + for i := range objects { - if err := n.DeleteObject(ctx, bucket, objects[i]); err != nil { + if err := n.deleteObject(ctx, bkt, objects[i]); err != nil { errs = append(errs, err) } } @@ -461,6 +548,14 @@ func (n *layer) DeleteBucket(ctx context.Context, p *DeleteBucketParams) error { return err } + ids, err := n.objectSearch(ctx, &findParams{cid: bucketInfo.CID}) + if err != nil { + return err + } + if len(ids) != 0 { + return api.GetAPIError(api.ErrBucketNotEmpty) + } + return n.deleteContainer(ctx, bucketInfo.CID) } @@ -581,10 +676,7 @@ func (n *layer) PutBucketVersioning(ctx context.Context, p *PutVersioningParams) } if objectInfo != nil { - addr := object.NewAddress() - addr.SetObjectID(objectInfo.ID()) - addr.SetContainerID(bucketInfo.CID) - if err = n.objectDelete(ctx, addr); err != nil { + if err = n.objectDelete(ctx, bucketInfo.CID, objectInfo.ID()); err != nil { return nil, err } } diff --git a/api/layer/object.go b/api/layer/object.go index 44b2258d..75bfab91 100644 --- a/api/layer/object.go +++ b/api/layer/object.go @@ -64,8 +64,9 @@ type ( ) const ( - versionsDelAttr = "S3-Versions-del" - versionsAddAttr = "S3-Versions-add" + versionsDelAttr = "S3-Versions-del" + versionsAddAttr = "S3-Versions-add" + versionsDeleteMarkAttr = "S3-Versions-delete-mark" ) // objectSearch returns all available objects by search params. @@ -121,11 +122,10 @@ func (n *layer) objectRange(ctx context.Context, p *getParams) ([]byte, error) { } // objectPut into NeoFS, took payload from io.Reader. -func (n *layer) objectPut(ctx context.Context, p *PutObjectParams) (*ObjectInfo, error) { +func (n *layer) objectPut(ctx context.Context, bkt *BucketInfo, p *PutObjectParams) (*ObjectInfo, error) { var ( err error obj string - bkt *BucketInfo own = n.Owner(ctx) ) @@ -136,9 +136,6 @@ func (n *layer) objectPut(ctx context.Context, p *PutObjectParams) (*ObjectInfo, if obj, err = url.QueryUnescape(p.Object); err != nil { return nil, err } - if bkt, err = n.GetBucketInfo(ctx, p.Bucket); err != nil { - return nil, err - } versioningEnabled := n.isVersioningEnabled(ctx, bkt) lastVersionInfo, err := n.headLastVersion(ctx, bkt, p.Object) @@ -155,15 +152,18 @@ func (n *layer) objectPut(ctx context.Context, p *PutObjectParams) (*ObjectInfo, versionsAddedStr += "," } versionsAddedStr += lastVersionInfo.ID().String() - addedVersions := object.NewAttribute() - addedVersions.SetKey(versionsAddAttr) - addedVersions.SetValue(versionsAddedStr) - attributes = append(attributes, addedVersions) - if delVersions := lastVersionInfo.Headers[versionsDelAttr]; len(delVersions) > 0 { - deletedVersions := object.NewAttribute() - deletedVersions.SetKey(versionsDelAttr) - deletedVersions.SetValue(delVersions) - attributes = append(attributes, deletedVersions) + p.Header[versionsAddAttr] = versionsAddedStr + + deleted := p.Header[versionsDelAttr] + if delVersions := lastVersionInfo.Headers[versionsDelAttr]; len(delVersions) != 0 { + if len(deleted) == 0 { + deleted = delVersions + } else { + deleted = delVersions + "," + deleted + } + } + if len(deleted) != 0 { + p.Header[versionsDelAttr] = deleted } } else { versionsDeletedStr := lastVersionInfo.Headers[versionsDelAttr] @@ -171,24 +171,19 @@ func (n *layer) objectPut(ctx context.Context, p *PutObjectParams) (*ObjectInfo, versionsDeletedStr += "," } versionsDeletedStr += lastVersionInfo.ID().String() - deletedVersions := object.NewAttribute() - deletedVersions.SetKey(versionsDelAttr) - deletedVersions.SetValue(versionsDeletedStr) + p.Header[versionsDelAttr] = versionsDeletedStr - attributes = append(attributes, deletedVersions) idsToDeleteArr = append(idsToDeleteArr, lastVersionInfo.ID()) } } - unix := strconv.FormatInt(time.Now().UTC().Unix(), 10) - filename := object.NewAttribute() filename.SetKey(object.AttributeFileName) filename.SetValue(obj) createdAt := object.NewAttribute() createdAt.SetKey(object.AttributeTimestamp) - createdAt.SetValue(unix) + createdAt.SetValue(strconv.FormatInt(time.Now().UTC().Unix(), 10)) attributes = append(attributes, filename, createdAt) @@ -240,11 +235,7 @@ func (n *layer) objectPut(ctx context.Context, p *PutObjectParams) (*ObjectInfo, } for _, id := range idsToDeleteArr { - addr := object.NewAddress() - addr.SetObjectID(id) - addr.SetContainerID(bkt.CID) - - if err = n.objectDelete(ctx, addr); err != nil { + if err = n.objectDelete(ctx, bkt.CID, id); err != nil { n.log.Warn("couldn't delete object", zap.Stringer("version id", id), zap.Error(err)) @@ -284,33 +275,28 @@ func (n *layer) headLastVersion(ctx context.Context, bkt *BucketInfo, objectName return objectInfoFromMeta(bkt, infos[len(infos)-1], "", ""), nil } -func (n *layer) headVersion(ctx context.Context, bkt *BucketInfo, objectName, versionID string) (*ObjectInfo, error) { - ids, err := n.objectSearch(ctx, &findParams{cid: bkt.CID, val: objectName}) - if err != nil { +func (n *layer) headVersion(ctx context.Context, bkt *BucketInfo, versionID string) (*ObjectInfo, error) { + oid := object.NewID() + if err := oid.Parse(versionID); err != nil { return nil, err } - if len(ids) == 0 { - return nil, api.GetAPIError(api.ErrNoSuchVersion) - } - for _, id := range ids { - if id.String() == versionID { - meta, err := n.objectHead(ctx, bkt.CID, id) - if err != nil { - if strings.Contains(err.Error(), "not found") { - return nil, api.GetAPIError(api.ErrNoSuchVersion) - } - return nil, err - } - return objectInfoFromMeta(bkt, meta, "", ""), nil + meta, err := n.objectHead(ctx, bkt.CID, oid) + if err != nil { + if strings.Contains(err.Error(), "not found") { + return nil, api.GetAPIError(api.ErrNoSuchVersion) } + return nil, err } - return nil, api.GetAPIError(api.ErrNoSuchVersion) + return objectInfoFromMeta(bkt, meta, "", ""), nil } // objectDelete puts tombstone object into neofs. -func (n *layer) objectDelete(ctx context.Context, address *object.Address) error { +func (n *layer) objectDelete(ctx context.Context, cid *cid.ID, oid *object.ID) error { + address := object.NewAddress() + address.SetContainerID(cid) + address.SetObjectID(oid) dop := new(client.DeleteObjectParams) dop.WithAddress(address) n.objCache.Delete(address) diff --git a/api/layer/util_test.go b/api/layer/util_test.go index 9b44d854..d393b143 100644 --- a/api/layer/util_test.go +++ b/api/layer/util_test.go @@ -48,6 +48,7 @@ func newTestInfo(oid *object.ID, bkt *BucketInfo, name string, isDir bool) *Obje id: oid, Name: name, Bucket: bkt.Name, + bucketID: bkt.CID, Size: defaultTestPayloadLength, ContentType: defaultTestContentType, Created: time.Unix(defaultTestCreated.Unix(), 0), From 43185de52aac5932bf1828c52fc0bd0fb3962af1 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Wed, 11 Aug 2021 13:02:13 +0300 Subject: [PATCH 06/10] [#122] Add list object versions Signed-off-by: Denis Kirillov --- api/handler/versioning.go | 3 +- api/headers.go | 2 +- api/layer/layer.go | 165 +++++++++++++++++++++++++++----------- api/layer/object.go | 4 +- api/layer/util.go | 9 ++- 5 files changed, 127 insertions(+), 56 deletions(-) diff --git a/api/handler/versioning.go b/api/handler/versioning.go index ece2e85c..19351a10 100644 --- a/api/handler/versioning.go +++ b/api/handler/versioning.go @@ -5,6 +5,7 @@ import ( "net/http" "github.com/nspcc-dev/neofs-s3-gw/api" + "github.com/nspcc-dev/neofs-s3-gw/api/errors" "github.com/nspcc-dev/neofs-s3-gw/api/layer" "go.uber.org/zap" ) @@ -14,7 +15,7 @@ func (h *handler) PutBucketVersioningHandler(w http.ResponseWriter, r *http.Requ configuration := new(VersioningConfiguration) if err := xml.NewDecoder(r.Body).Decode(configuration); err != nil { - h.logAndSendError(w, "couldn't decode versioning configuration", reqInfo, api.GetAPIError(api.ErrIllegalVersioningConfigurationException)) + h.logAndSendError(w, "couldn't decode versioning configuration", reqInfo, errors.GetAPIError(errors.ErrIllegalVersioningConfigurationException)) return } diff --git a/api/headers.go b/api/headers.go index adf81c5c..767eca43 100644 --- a/api/headers.go +++ b/api/headers.go @@ -4,7 +4,7 @@ package api const ( MetadataPrefix = "X-Amz-Meta-" AmzMetadataDirective = "X-Amz-Metadata-Directive" - AmzVersionID = "X-Amz-Version-Id" + AmzVersionID = "X-Amz-Version-Id" LastModified = "Last-Modified" Date = "Date" diff --git a/api/layer/layer.go b/api/layer/layer.go index a0a45a50..31773fec 100644 --- a/api/layer/layer.go +++ b/api/layer/layer.go @@ -322,15 +322,15 @@ func (n *layer) GetObject(ctx context.Context, p *GetObjectParams) error { return nil } -func (n *layer) checkObject(ctx context.Context, cid *cid.ID, filename string) error { - var err error - - if _, err = n.objectFindID(ctx, &findParams{cid: cid, val: filename}); err == nil { - return new(errors.ObjectAlreadyExists) - } - - return err -} +//func (n *layer) checkObject(ctx context.Context, cid *cid.ID, filename string) error { +// var err error +// +// if _, err = n.objectFindID(ctx, &findParams{cid: cid, val: filename}); err == nil { +// return new(errors.ObjectAlreadyExists) +// } +// +// return err +//} // GetObjectInfo returns meta information about the object. func (n *layer) GetObjectInfo(ctx context.Context, p *HeadObjectParams) (*ObjectInfo, error) { @@ -344,7 +344,7 @@ func (n *layer) GetObjectInfo(ctx context.Context, p *HeadObjectParams) (*Object objInfo, err := n.headLastVersion(ctx, bkt, p.Object) if err == nil { if deleteMark, err2 := strconv.ParseBool(objInfo.Headers[versionsDeleteMarkAttr]); err2 == nil && deleteMark { - return nil, api.GetAPIError(api.ErrNoSuchKey) + return nil, errors.GetAPIError(errors.ErrNoSuchKey) } } return objInfo, err @@ -423,7 +423,7 @@ func (n *layer) deleteObject(ctx context.Context, bkt *BucketInfo, obj *Versione ) versioningEnabled := n.isVersioningEnabled(ctx, bkt) - if !versioningEnabled && obj.VersionID != "null" && obj.VersionID != "" { + if !versioningEnabled && obj.VersionID != unversionedObjectVersionID && obj.VersionID != "" { return errors.GetAPIError(errors.ErrInvalidVersion) } @@ -431,17 +431,17 @@ func (n *layer) deleteObject(ctx context.Context, bkt *BucketInfo, obj *Versione if len(obj.VersionID) != 0 { id := object.NewID() if err := id.Parse(obj.VersionID); err != nil { - return &errors.DeleteError{Err: api.GetAPIError(api.ErrInvalidVersion), Object: obj.String()} + return &errors.DeleteError{Err: errors.GetAPIError(errors.ErrInvalidVersion), Object: obj.String()} } ids = []*object.ID{id} lastObject, err := n.headLastVersion(ctx, bkt, obj.Name) if err != nil { - return &api.DeleteError{Err: err, Object: obj.String()} + return &errors.DeleteError{Err: err, Object: obj.String()} } if !strings.Contains(lastObject.Headers[versionsAddAttr], obj.VersionID) || strings.Contains(lastObject.Headers[versionsDelAttr], obj.VersionID) { - return &api.DeleteError{Err: api.GetAPIError(api.ErrInvalidVersion), Object: obj.String()} + return &errors.DeleteError{Err: errors.GetAPIError(errors.ErrInvalidVersion), Object: obj.String()} } if lastObject.ID().String() == obj.VersionID { @@ -449,7 +449,7 @@ func (n *layer) deleteObject(ctx context.Context, bkt *BucketInfo, obj *Versione addedVersions := strings.Split(added, ",") sourceCopyVersion, err := n.headVersion(ctx, bkt, addedVersions[len(addedVersions)-1]) if err != nil { - return &api.DeleteError{Err: err, Object: obj.String()} + return &errors.DeleteError{Err: err, Object: obj.String()} } p := &CopyObjectParams{ SrcObject: sourceCopyVersion, @@ -471,7 +471,7 @@ func (n *layer) deleteObject(ctx context.Context, bkt *BucketInfo, obj *Versione }, } if _, err := n.objectPut(ctx, bkt, p); err != nil { - return &api.DeleteError{Err: err, Object: obj.String()} + return &errors.DeleteError{Err: err, Object: obj.String()} } } } else { @@ -553,7 +553,7 @@ func (n *layer) DeleteBucket(ctx context.Context, p *DeleteBucketParams) error { return err } if len(ids) != 0 { - return api.GetAPIError(api.ErrBucketNotEmpty) + return errors.GetAPIError(errors.ErrBucketNotEmpty) } return n.deleteContainer(ctx, bucketInfo.CID) @@ -561,11 +561,11 @@ func (n *layer) DeleteBucket(ctx context.Context, p *DeleteBucketParams) error { func (n *layer) ListObjectVersions(ctx context.Context, p *ListObjectVersionsParams) (*ListObjectVersionsInfo, error) { var ( - res = ListObjectVersionsInfo{} - err error - bkt *BucketInfo - ids []*object.ID - uniqNames = make(map[string]bool) + res = ListObjectVersionsInfo{} + err error + bkt *BucketInfo + ids []*object.ID + latest = make(map[string]*ObjectVersionInfo) ) if bkt, err = n.GetBucketInfo(ctx, p.Bucket); err != nil { @@ -575,9 +575,9 @@ func (n *layer) ListObjectVersions(ctx context.Context, p *ListObjectVersionsPar } versions := make([]*ObjectVersionInfo, 0, len(ids)) - // todo: deletemarkers is empty now, we will use it after proper realization of versioning deleted := make([]*DeletedObjectInfo, 0, len(ids)) - res.DeleteMarker = deleted + + deletedVersions := []string{} for _, id := range ids { meta, err := n.objectHead(ctx, bkt.CID, id) @@ -586,45 +586,114 @@ func (n *layer) ListObjectVersions(ctx context.Context, p *ListObjectVersionsPar continue } if ov := objectVersionInfoFromMeta(bkt, meta, p.Prefix, p.Delimiter); ov != nil { - if _, ok := uniqNames[ov.Object.Name]; ok { + if ov.Object.Name <= p.KeyMarker { continue } - if len(p.KeyMarker) > 0 && ov.Object.Name <= p.KeyMarker { - continue + if currentLatest, ok := latest[ov.Object.Name]; ok { + if less(currentLatest, ov) { + latest[ov.Object.Name] = ov + } + } else { + latest[ov.Object.Name] = ov + } + + if del := ov.Object.Headers[versionsDelAttr]; len(del) != 0 { + deletedVersions = append(deletedVersions, strings.Split(del, ",")...) + } + + if parsed, err := strconv.ParseBool(ov.Object.Headers[versionsDeleteMarkAttr]); err == nil && parsed { + deleted = append(deleted, &DeletedObjectInfo{ + Owner: ov.Object.Owner, + Key: ov.Object.Name, + VersionID: ov.VersionID, + LastModified: ov.Object.Created.Format(time.RFC3339), + }) + } else { + versions = append(versions, ov) } - uniqNames[ov.Object.Name] = ov.Object.isDir - versions = append(versions, ov) } } sort.Slice(versions, func(i, j int) bool { + if contains(deletedVersions, versions[i].VersionID) { + return true + } + if contains(deletedVersions, versions[j].VersionID) { + return false + } + if versions[i].Object.Name == versions[j].Object.Name { + if versions[i].CreationEpoch == versions[j].CreationEpoch { + return versions[i].VersionID < versions[j].VersionID + } + return versions[i].CreationEpoch < versions[j].CreationEpoch + } return versions[i].Object.Name < versions[j].Object.Name }) - if len(versions) > p.MaxKeys { - res.IsTruncated = true - - lastVersion := versions[p.MaxKeys-1] - res.KeyMarker = lastVersion.Object.Name - res.VersionIDMarker = lastVersion.VersionID - - nextVersion := versions[p.MaxKeys] - res.NextKeyMarker = nextVersion.Object.Name - res.NextVersionIDMarker = nextVersion.VersionID - - versions = versions[:p.MaxKeys] - } - - for _, ov := range versions { - if isDir := uniqNames[ov.Object.Name]; isDir { - res.CommonPrefixes = append(res.CommonPrefixes, &ov.Object.Name) - } else { - res.Version = append(res.Version, ov) + for i, objVersion := range versions { + if i == len(versions)-1 || objVersion.Object.Name != versions[i+1].Object.Name { + objVersion.IsLatest = true } } + + for i, objVersion := range versions { + if !contains(deletedVersions, objVersion.VersionID) { + versions = versions[i:] + break + } + } + + for i, objVersion := range deleted { + if !contains(deletedVersions, objVersion.VersionID) { + deleted = deleted[i:] + break + } + } + + //if len(versions) > p.MaxKeys { + // res.IsTruncated = true + // + // lastVersion := versions[p.MaxKeys-1] + // res.KeyMarker = lastVersion.Object.Name + // res.VersionIDMarker = lastVersion.VersionID + // + // nextVersion := versions[p.MaxKeys] + // res.NextKeyMarker = nextVersion.Object.Name + // res.NextVersionIDMarker = nextVersion.VersionID + // + // versions = versions[:p.MaxKeys] + //} + // + //for _, ov := range versions { + // if isDir := uniqNames[ov.Object.Name]; isDir { + // res.CommonPrefixes = append(res.CommonPrefixes, &ov.Object.Name) + // } else { + // res.Version = append(res.Version, ov) + // } + //} + + res.Version = versions + res.DeleteMarker = deleted + return &res, nil } +func less(ov1, ov2 *ObjectVersionInfo) bool { + if ov1.CreationEpoch == ov2.CreationEpoch { + return ov1.VersionID < ov2.VersionID + } + return ov1.CreationEpoch < ov2.CreationEpoch +} + +func contains(list []string, elem string) bool { + for _, item := range list { + if elem == item { + return true + } + } + return false +} + func (n *layer) PutBucketVersioning(ctx context.Context, p *PutVersioningParams) (*ObjectInfo, error) { bucketInfo, err := n.GetBucketInfo(ctx, p.Bucket) if err != nil { diff --git a/api/layer/object.go b/api/layer/object.go index 75bfab91..0edf92b8 100644 --- a/api/layer/object.go +++ b/api/layer/object.go @@ -252,7 +252,7 @@ func (n *layer) headLastVersion(ctx context.Context, bkt *BucketInfo, objectName } if len(ids) == 0 { - return nil, api.GetAPIError(api.ErrNoSuchKey) + return nil, apiErrors.GetAPIError(apiErrors.ErrNoSuchKey) } infos := make([]*object.Object, 0, len(ids)) @@ -284,7 +284,7 @@ func (n *layer) headVersion(ctx context.Context, bkt *BucketInfo, versionID stri meta, err := n.objectHead(ctx, bkt.CID, oid) if err != nil { if strings.Contains(err.Error(), "not found") { - return nil, api.GetAPIError(api.ErrNoSuchVersion) + return nil, apiErrors.GetAPIError(apiErrors.ErrNoSuchVersion) } return nil, err } diff --git a/api/layer/util.go b/api/layer/util.go index 990c00ac..6fce6919 100644 --- a/api/layer/util.go +++ b/api/layer/util.go @@ -54,9 +54,10 @@ type ( // ObjectVersionInfo stores info about objects versions. ObjectVersionInfo struct { - Object *ObjectInfo - IsLatest bool - VersionID string + Object *ObjectInfo + IsLatest bool + VersionID string + CreationEpoch uint64 } // DeletedObjectInfo stores info about deleted versions of objects. @@ -156,7 +157,7 @@ func objectVersionInfoFromMeta(bkt *BucketInfo, meta *object.Object, prefix, del if oi == nil { return nil } - return &ObjectVersionInfo{Object: oi, IsLatest: true, VersionID: unversionedObjectVersionID} + return &ObjectVersionInfo{Object: oi, VersionID: meta.ID().String(), CreationEpoch: meta.CreationEpoch()} } func filenameFromObject(o *object.Object) string { From 9c058a70fdfc0d930a9bc80d5bff17fb50f29aa3 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 13 Aug 2021 17:13:14 +0300 Subject: [PATCH 07/10] [#122] Add tests Signed-off-by: Denis Kirillov --- api/handler/object_list.go | 12 +- api/layer/layer.go | 349 ++++++++++---------- api/layer/object.go | 248 ++++++++------ api/layer/util.go | 75 ++--- api/layer/versioning_test.go | 608 +++++++++++++++++++++++++++++++++++ 5 files changed, 960 insertions(+), 332 deletions(-) create mode 100644 api/layer/versioning_test.go diff --git a/api/handler/object_list.go b/api/handler/object_list.go index ef8f0dd4..d0233e98 100644 --- a/api/handler/object_list.go +++ b/api/handler/object_list.go @@ -276,7 +276,7 @@ func encodeListObjectVersionsToResponse(info *layer.ListObjectVersionsInfo, buck DisplayName: ver.Object.Owner.String(), }, Size: ver.Object.Size, - VersionID: ver.VersionID, + VersionID: ver.Object.Version(), ETag: ver.Object.HashSum, }) } @@ -284,13 +284,13 @@ func encodeListObjectVersionsToResponse(info *layer.ListObjectVersionsInfo, buck for _, del := range info.DeleteMarker { res.DeleteMarker = append(res.DeleteMarker, DeleteMarkerEntry{ IsLatest: del.IsLatest, - Key: del.Key, - LastModified: del.LastModified, + Key: del.Object.Name, + LastModified: del.Object.Created.Format(time.RFC3339), Owner: Owner{ - ID: del.Owner.String(), - DisplayName: del.Owner.String(), + ID: del.Object.Owner.String(), + DisplayName: del.Object.Owner.String(), }, - VersionID: del.VersionID, + VersionID: del.Object.Version(), }) } diff --git a/api/layer/layer.go b/api/layer/layer.go index 31773fec..fbd915fe 100644 --- a/api/layer/layer.go +++ b/api/layer/layer.go @@ -136,6 +136,13 @@ type ( VersionID string } + objectVersions struct { + objects []*ObjectInfo + addList []string + delList []string + isSorted bool + } + // NeoFS provides basic NeoFS interface. NeoFS interface { Get(ctx context.Context, address *object.Address) (*object.Object, error) @@ -170,9 +177,66 @@ type ( } ) +func (v *objectVersions) appendVersion(oi *ObjectInfo) { + addVers := append(splitVersions(oi.Headers[versionsAddAttr]), oi.Version()) + delVers := splitVersions(oi.Headers[versionsDelAttr]) + v.objects = append(v.objects, oi) + for _, add := range addVers { + if !contains(v.addList, add) { + v.addList = append(v.addList, add) + } + } + for _, del := range delVers { + if !contains(v.delList, del) { + v.delList = append(v.delList, del) + } + } + v.isSorted = false +} + +func (v *objectVersions) sort() { + if !v.isSorted { + sortVersions(v.objects) + v.isSorted = true + } +} + +func (v *objectVersions) getLast() *ObjectInfo { + if len(v.objects) == 0 { + return nil + } + + v.sort() + existedVersions := getExistedVersions(v) + for i := len(v.objects) - 1; i >= 0; i-- { + if contains(existedVersions, v.objects[i].Version()) { + delMarkHeader := v.objects[i].Headers[versionsDeleteMarkAttr] + if delMarkHeader == "" { + return v.objects[i] + } + if delMarkHeader == "*" { + return nil + } + } + } + + return nil +} + +func (v *objectVersions) getAddHeader() string { + return strings.Join(v.addList, ",") +} + +func (v *objectVersions) getDelHeader() string { + return strings.Join(v.delList, ",") +} + const ( - unversionedObjectVersionID = "null" - bktVersionSettingsObject = ".s3-versioning-settings" + unversionedObjectVersionID = "null" + bktVersionSettingsObject = ".s3-versioning-settings" + objectSystemAttributeName = "S3-System-name" + attrVersionsIgnore = "S3-Versions-ignore" + attrSettingsVersioningEnabled = "S3-Settings-Versioning-enabled" ) func (t *VersionedObject) String() string { @@ -322,16 +386,6 @@ func (n *layer) GetObject(ctx context.Context, p *GetObjectParams) error { return nil } -//func (n *layer) checkObject(ctx context.Context, cid *cid.ID, filename string) error { -// var err error -// -// if _, err = n.objectFindID(ctx, &findParams{cid: cid, val: filename}); err == nil { -// return new(errors.ObjectAlreadyExists) -// } -// -// return err -//} - // GetObjectInfo returns meta information about the object. func (n *layer) GetObjectInfo(ctx context.Context, p *HeadObjectParams) (*ObjectInfo, error) { bkt, err := n.GetBucketInfo(ctx, p.Bucket) @@ -341,20 +395,14 @@ func (n *layer) GetObjectInfo(ctx context.Context, p *HeadObjectParams) (*Object } if len(p.VersionID) == 0 { - objInfo, err := n.headLastVersion(ctx, bkt, p.Object) - if err == nil { - if deleteMark, err2 := strconv.ParseBool(objInfo.Headers[versionsDeleteMarkAttr]); err2 == nil && deleteMark { - return nil, errors.GetAPIError(errors.ErrNoSuchKey) - } - } - return objInfo, err + return n.headLastVersionIfNotDeleted(ctx, bkt, p.Object) } return n.headVersion(ctx, bkt, p.VersionID) } func (n *layer) getSettingsObjectInfo(ctx context.Context, bkt *BucketInfo) (*ObjectInfo, error) { - oid, err := n.objectFindID(ctx, &findParams{cid: bkt.CID, val: bktVersionSettingsObject}) + oid, err := n.objectFindID(ctx, &findParams{cid: bkt.CID, attr: objectSystemAttributeName, val: bktVersionSettingsObject}) if err != nil { return nil, err } @@ -428,73 +476,24 @@ func (n *layer) deleteObject(ctx context.Context, bkt *BucketInfo, obj *Versione } if versioningEnabled { + p := &PutObjectParams{ + Object: obj.Name, + Reader: bytes.NewReader(nil), + Header: map[string]string{versionsDeleteMarkAttr: obj.VersionID}, + } if len(obj.VersionID) != 0 { - id := object.NewID() - if err := id.Parse(obj.VersionID); err != nil { - return &errors.DeleteError{Err: errors.GetAPIError(errors.ErrInvalidVersion), Object: obj.String()} + id, err := n.checkVersionsExists(ctx, bkt, obj) + if err != nil { + return err } ids = []*object.ID{id} - lastObject, err := n.headLastVersion(ctx, bkt, obj.Name) - if err != nil { - return &errors.DeleteError{Err: err, Object: obj.String()} - } - if !strings.Contains(lastObject.Headers[versionsAddAttr], obj.VersionID) || - strings.Contains(lastObject.Headers[versionsDelAttr], obj.VersionID) { - return &errors.DeleteError{Err: errors.GetAPIError(errors.ErrInvalidVersion), Object: obj.String()} - } - - if lastObject.ID().String() == obj.VersionID { - if added := lastObject.Headers[versionsAddAttr]; len(added) > 0 { - addedVersions := strings.Split(added, ",") - sourceCopyVersion, err := n.headVersion(ctx, bkt, addedVersions[len(addedVersions)-1]) - if err != nil { - return &errors.DeleteError{Err: err, Object: obj.String()} - } - p := &CopyObjectParams{ - SrcObject: sourceCopyVersion, - DstBucket: bkt.Name, - DstObject: obj.Name, - SrcSize: sourceCopyVersion.Size, - Header: map[string]string{versionsDelAttr: obj.VersionID}, - } - if _, err := n.CopyObject(ctx, p); err != nil { - return err - } - } else { - p := &PutObjectParams{ - Object: obj.Name, - Reader: bytes.NewReader(nil), - Header: map[string]string{ - versionsDelAttr: obj.VersionID, - versionsDeleteMarkAttr: strconv.FormatBool(true), - }, - } - if _, err := n.objectPut(ctx, bkt, p); err != nil { - return &errors.DeleteError{Err: err, Object: obj.String()} - } - } - } else { - p := &CopyObjectParams{ - SrcObject: lastObject, - DstBucket: bkt.Name, - DstObject: obj.Name, - SrcSize: lastObject.Size, - Header: map[string]string{versionsDelAttr: obj.VersionID}, - } - if _, err := n.CopyObject(ctx, p); err != nil { - return err - } - } + p.Header[versionsDelAttr] = obj.VersionID } else { - p := &PutObjectParams{ - Object: obj.Name, - Reader: bytes.NewReader(nil), - Header: map[string]string{versionsDeleteMarkAttr: strconv.FormatBool(true)}, - } - if _, err := n.objectPut(ctx, bkt, p); err != nil { - return &errors.DeleteError{Err: err, Object: obj.String()} - } + p.Header[versionsDeleteMarkAttr] = "*" + } + if _, err = n.objectPut(ctx, bkt, p); err != nil { + return &errors.DeleteError{Err: err, Object: obj.String()} } } else { ids, err = n.objectSearch(ctx, &findParams{cid: bkt.CID, val: obj.Name}) @@ -512,6 +511,23 @@ func (n *layer) deleteObject(ctx context.Context, bkt *BucketInfo, obj *Versione return nil } +func (n *layer) checkVersionsExists(ctx context.Context, bkt *BucketInfo, obj *VersionedObject) (*object.ID, error) { + id := object.NewID() + if err := id.Parse(obj.VersionID); err != nil { + return nil, &errors.DeleteError{Err: errors.GetAPIError(errors.ErrInvalidVersion), Object: obj.String()} + } + + versions, err := n.headVersions(ctx, bkt, obj.Name) + if err != nil { + return nil, &errors.DeleteError{Err: err, Object: obj.String()} + } + if !contains(getExistedVersions(versions), obj.VersionID) { + return nil, &errors.DeleteError{Err: errors.GetAPIError(errors.ErrInvalidVersion), Object: obj.String()} + } + + return id, nil +} + // DeleteObjects from the storage. func (n *layer) DeleteObjects(ctx context.Context, bucket string, objects []*VersionedObject) []error { var errs = make([]error, 0, len(objects)) @@ -560,24 +576,17 @@ func (n *layer) DeleteBucket(ctx context.Context, p *DeleteBucketParams) error { } func (n *layer) ListObjectVersions(ctx context.Context, p *ListObjectVersionsParams) (*ListObjectVersionsInfo, error) { - var ( - res = ListObjectVersionsInfo{} - err error - bkt *BucketInfo - ids []*object.ID - latest = make(map[string]*ObjectVersionInfo) - ) + res := ListObjectVersionsInfo{} + versions := make(map[string]*objectVersions) - if bkt, err = n.GetBucketInfo(ctx, p.Bucket); err != nil { - return nil, err - } else if ids, err = n.objectSearch(ctx, &findParams{cid: bkt.CID}); err != nil { + bkt, err := n.GetBucketInfo(ctx, p.Bucket) + if err != nil { + return nil, err + } + ids, err := n.objectSearch(ctx, &findParams{cid: bkt.CID}) + if err != nil { return nil, err } - - versions := make([]*ObjectVersionInfo, 0, len(ids)) - deleted := make([]*DeletedObjectInfo, 0, len(ids)) - - deletedVersions := []string{} for _, id := range ids { meta, err := n.objectHead(ctx, bkt.CID, id) @@ -585,102 +594,80 @@ func (n *layer) ListObjectVersions(ctx context.Context, p *ListObjectVersionsPar n.log.Warn("could not fetch object meta", zap.Error(err)) continue } - if ov := objectVersionInfoFromMeta(bkt, meta, p.Prefix, p.Delimiter); ov != nil { - if ov.Object.Name <= p.KeyMarker { + if oi := objectInfoFromMeta(bkt, meta, p.Prefix, p.Delimiter); oi != nil { + if oi.Name <= p.KeyMarker { continue } - if currentLatest, ok := latest[ov.Object.Name]; ok { - if less(currentLatest, ov) { - latest[ov.Object.Name] = ov - } - } else { - latest[ov.Object.Name] = ov + if isSystem(oi) { + continue } - if del := ov.Object.Headers[versionsDelAttr]; len(del) != 0 { - deletedVersions = append(deletedVersions, strings.Split(del, ",")...) - } - - if parsed, err := strconv.ParseBool(ov.Object.Headers[versionsDeleteMarkAttr]); err == nil && parsed { - deleted = append(deleted, &DeletedObjectInfo{ - Owner: ov.Object.Owner, - Key: ov.Object.Name, - VersionID: ov.VersionID, - LastModified: ov.Object.Created.Format(time.RFC3339), - }) + if objVersions, ok := versions[oi.Name]; ok { + objVersions.appendVersion(oi) + versions[oi.Name] = objVersions } else { - versions = append(versions, ov) + objVersion := &objectVersions{} + objVersion.appendVersion(oi) + versions[oi.Name] = objVersion } } } - sort.Slice(versions, func(i, j int) bool { - if contains(deletedVersions, versions[i].VersionID) { - return true - } - if contains(deletedVersions, versions[j].VersionID) { - return false - } - if versions[i].Object.Name == versions[j].Object.Name { - if versions[i].CreationEpoch == versions[j].CreationEpoch { - return versions[i].VersionID < versions[j].VersionID - } - return versions[i].CreationEpoch < versions[j].CreationEpoch - } - return versions[i].Object.Name < versions[j].Object.Name + for _, v := range versions { + existed, deleted := triageVersions(v) + res.Version = append(res.Version, existed...) + res.DeleteMarker = append(res.DeleteMarker, deleted...) + } + + sort.Slice(res.Version, func(i, j int) bool { + return res.Version[i].Object.Name < res.Version[j].Object.Name + }) + sort.Slice(res.DeleteMarker, func(i, j int) bool { + return res.DeleteMarker[i].Object.Name < res.DeleteMarker[j].Object.Name }) - - for i, objVersion := range versions { - if i == len(versions)-1 || objVersion.Object.Name != versions[i+1].Object.Name { - objVersion.IsLatest = true - } - } - - for i, objVersion := range versions { - if !contains(deletedVersions, objVersion.VersionID) { - versions = versions[i:] - break - } - } - - for i, objVersion := range deleted { - if !contains(deletedVersions, objVersion.VersionID) { - deleted = deleted[i:] - break - } - } - - //if len(versions) > p.MaxKeys { - // res.IsTruncated = true - // - // lastVersion := versions[p.MaxKeys-1] - // res.KeyMarker = lastVersion.Object.Name - // res.VersionIDMarker = lastVersion.VersionID - // - // nextVersion := versions[p.MaxKeys] - // res.NextKeyMarker = nextVersion.Object.Name - // res.NextVersionIDMarker = nextVersion.VersionID - // - // versions = versions[:p.MaxKeys] - //} - // - //for _, ov := range versions { - // if isDir := uniqNames[ov.Object.Name]; isDir { - // res.CommonPrefixes = append(res.CommonPrefixes, &ov.Object.Name) - // } else { - // res.Version = append(res.Version, ov) - // } - //} - - res.Version = versions - res.DeleteMarker = deleted return &res, nil } -func less(ov1, ov2 *ObjectVersionInfo) bool { +func sortVersions(versions []*ObjectInfo) { + sort.Slice(versions, func(i, j int) bool { + return less(versions[i], versions[j]) + }) +} + +func triageVersions(objectVersions *objectVersions) ([]*ObjectVersionInfo, []*ObjectVersionInfo) { + if objectVersions == nil || len(objectVersions.objects) == 0 { + return nil, nil + } + + sortVersions(objectVersions.objects) + + var resVersion []*ObjectVersionInfo + var resDelMarkVersions []*ObjectVersionInfo + + isLatest := true + for i := len(objectVersions.objects) - 1; i >= 0; i-- { + version := objectVersions.objects[i] + if contains(objectVersions.delList, version.Version()) { + continue + } + + ovi := &ObjectVersionInfo{Object: version, IsLatest: isLatest} + isLatest = false + + if len(version.Headers[versionsDeleteMarkAttr]) == 0 { + resVersion = append(resVersion, ovi) + } else { + resDelMarkVersions = append(resDelMarkVersions, ovi) + } + } + + return resVersion, resDelMarkVersions +} + +func less(ov1, ov2 *ObjectInfo) bool { if ov1.CreationEpoch == ov2.CreationEpoch { - return ov1.VersionID < ov2.VersionID + return ov1.Version() < ov2.Version() } return ov1.CreationEpoch < ov2.CreationEpoch } @@ -711,7 +698,7 @@ func (n *layer) PutBucketVersioning(ctx context.Context, p *PutVersioningParams) attributes := make([]*object.Attribute, 0, 3) filename := object.NewAttribute() - filename.SetKey(object.AttributeFileName) + filename.SetKey(objectSystemAttributeName) filename.SetValue(bktVersionSettingsObject) createdAt := object.NewAttribute() @@ -719,11 +706,11 @@ func (n *layer) PutBucketVersioning(ctx context.Context, p *PutVersioningParams) createdAt.SetValue(strconv.FormatInt(time.Now().UTC().Unix(), 10)) versioningIgnore := object.NewAttribute() - versioningIgnore.SetKey("S3-Versions-ignore") + versioningIgnore.SetKey(attrVersionsIgnore) versioningIgnore.SetValue(strconv.FormatBool(true)) settingsVersioningEnabled := object.NewAttribute() - settingsVersioningEnabled.SetKey("S3-Settings-Versioning-enabled") + settingsVersioningEnabled.SetKey(attrSettingsVersioningEnabled) settingsVersioningEnabled.SetValue(strconv.FormatBool(p.Settings.VersioningEnabled)) attributes = append(attributes, filename, createdAt, versioningIgnore, settingsVersioningEnabled) diff --git a/api/layer/object.go b/api/layer/object.go index 0edf92b8..b77c3589 100644 --- a/api/layer/object.go +++ b/api/layer/object.go @@ -3,7 +3,6 @@ package layer import ( "context" "errors" - "fmt" "io" "net/url" "sort" @@ -14,14 +13,16 @@ import ( "github.com/nspcc-dev/neofs-api-go/pkg/client" cid "github.com/nspcc-dev/neofs-api-go/pkg/container/id" "github.com/nspcc-dev/neofs-api-go/pkg/object" + "github.com/nspcc-dev/neofs-api-go/pkg/owner" apiErrors "github.com/nspcc-dev/neofs-s3-gw/api/errors" "go.uber.org/zap" ) type ( findParams struct { - val string - cid *cid.ID + attr string + val string + cid *cid.ID } getParams struct { @@ -78,7 +79,11 @@ func (n *layer) objectSearch(ctx context.Context, p *findParams) ([]*object.ID, if filename, err := url.QueryUnescape(p.val); err != nil { return nil, err } else if filename != "" { - opts.AddFilter(object.AttributeFileName, filename, object.MatchStringEqual) + if p.attr == "" { + opts.AddFilter(object.AttributeFileName, filename, object.MatchStringEqual) + } else { + opts.AddFilter(p.attr, filename, object.MatchStringEqual) + } } return n.pool.SearchObject(ctx, new(client.SearchObjectParams).WithContainerID(p.cid).WithSearchFilters(opts), n.BearerOpt(ctx)) } @@ -123,60 +128,59 @@ func (n *layer) objectRange(ctx context.Context, p *getParams) ([]byte, error) { // objectPut into NeoFS, took payload from io.Reader. func (n *layer) objectPut(ctx context.Context, bkt *BucketInfo, p *PutObjectParams) (*ObjectInfo, error) { - var ( - err error - obj string - own = n.Owner(ctx) - ) - - if p.Object == bktVersionSettingsObject { - return nil, fmt.Errorf("trying put bucket settings object") - } - - if obj, err = url.QueryUnescape(p.Object); err != nil { + own := n.Owner(ctx) + obj, err := url.QueryUnescape(p.Object) + if err != nil { return nil, err } versioningEnabled := n.isVersioningEnabled(ctx, bkt) - lastVersionInfo, err := n.headLastVersion(ctx, bkt, p.Object) + versions, err := n.headVersions(ctx, bkt, obj) if err != nil && !apiErrors.IsS3Error(err, apiErrors.ErrNoSuchKey) { return nil, err } + idsToDeleteArr := updateCRDT2PSetHeaders(p, versions, versioningEnabled) - attributes := make([]*object.Attribute, 0, len(p.Header)+1) - var idsToDeleteArr []*object.ID - if lastVersionInfo != nil { - if versioningEnabled { - versionsAddedStr := lastVersionInfo.Headers[versionsAddAttr] - if len(versionsAddedStr) != 0 { - versionsAddedStr += "," - } - versionsAddedStr += lastVersionInfo.ID().String() - p.Header[versionsAddAttr] = versionsAddedStr + rawObject := formRawObject(p, bkt.CID, own, obj) + r := newDetector(p.Reader) - deleted := p.Header[versionsDelAttr] - if delVersions := lastVersionInfo.Headers[versionsDelAttr]; len(delVersions) != 0 { - if len(deleted) == 0 { - deleted = delVersions - } else { - deleted = delVersions + "," + deleted - } - } - if len(deleted) != 0 { - p.Header[versionsDelAttr] = deleted - } - } else { - versionsDeletedStr := lastVersionInfo.Headers[versionsDelAttr] - if len(versionsDeletedStr) != 0 { - versionsDeletedStr += "," - } - versionsDeletedStr += lastVersionInfo.ID().String() - p.Header[versionsDelAttr] = versionsDeletedStr + ops := new(client.PutObjectParams).WithObject(rawObject.Object()).WithPayloadReader(r) + oid, err := n.pool.PutObject(ctx, ops, n.BearerOpt(ctx)) + if err != nil { + return nil, err + } - idsToDeleteArr = append(idsToDeleteArr, lastVersionInfo.ID()) + meta, err := n.objectHead(ctx, bkt.CID, oid) + if err != nil { + return nil, err + } + + for _, id := range idsToDeleteArr { + if err = n.objectDelete(ctx, bkt.CID, id); err != nil { + n.log.Warn("couldn't delete object", + zap.Stringer("version id", id), + zap.Error(err)) } } + return &ObjectInfo{ + id: oid, + bucketID: bkt.CID, + + Owner: own, + Bucket: p.Bucket, + Name: p.Object, + Size: p.Size, + Created: time.Now(), + CreationEpoch: meta.CreationEpoch(), + Headers: p.Header, + ContentType: r.contentType, + HashSum: meta.PayloadChecksum().String(), + }, nil +} + +func formRawObject(p *PutObjectParams, bktID *cid.ID, own *owner.ID, obj string) *object.RawObject { + attributes := make([]*object.Attribute, 0, len(p.Header)+2) filename := object.NewAttribute() filename.SetKey(object.AttributeFileName) filename.SetValue(obj) @@ -197,55 +201,68 @@ func (n *layer) objectPut(ctx context.Context, bkt *BucketInfo, p *PutObjectPara raw := object.NewRaw() raw.SetOwnerID(own) - raw.SetContainerID(bkt.CID) + raw.SetContainerID(bktID) raw.SetAttributes(attributes...) - r := newDetector(p.Reader) + return raw +} - ops := new(client.PutObjectParams).WithObject(raw.Object()).WithPayloadReader(r) - oid, err := n.pool.PutObject( - ctx, - ops, - n.BearerOpt(ctx), - ) - if err != nil { - return nil, err +func updateCRDT2PSetHeaders(p *PutObjectParams, versions *objectVersions, versioningEnabled bool) []*object.ID { + var idsToDeleteArr []*object.ID + if versions == nil { + return idsToDeleteArr } - meta, err := n.objectHead(ctx, bkt.CID, oid) - if err != nil { - return nil, err - } + if versioningEnabled { + if len(versions.addList) != 0 { + p.Header[versionsAddAttr] = versions.getAddHeader() + } - if err = n.objCache.Put(addr, *meta); err != nil { - n.log.Error("couldn't cache an object", zap.Error(err)) - } + deleted := versions.getDelHeader() + // p.Header[versionsDelAttr] can be not empty when deleting specific version + if delAttr := p.Header[versionsDelAttr]; len(delAttr) != 0 { + if len(deleted) != 0 { + p.Header[versionsDelAttr] = deleted + "," + delAttr + } else { + p.Header[versionsDelAttr] = delAttr + } + } else if len(deleted) != 0 { + p.Header[versionsDelAttr] = deleted + } + } else { + versionsDeletedStr := versions.getDelHeader() + if len(versionsDeletedStr) != 0 { + versionsDeletedStr += "," + } - objInfo := &ObjectInfo{ - id: oid, + lastVersion := versions.getLast() + p.Header[versionsDelAttr] = versionsDeletedStr + lastVersion.Version() + idsToDeleteArr = append(idsToDeleteArr, lastVersion.ID()) - Owner: own, - Bucket: p.Bucket, - Name: p.Object, - Size: p.Size, - Created: time.Now(), - Headers: p.Header, - ContentType: r.contentType, - HashSum: meta.PayloadChecksum().String(), - } - - for _, id := range idsToDeleteArr { - if err = n.objectDelete(ctx, bkt.CID, id); err != nil { - n.log.Warn("couldn't delete object", - zap.Stringer("version id", id), - zap.Error(err)) + for _, version := range versions.objects { + if contains(versions.delList, version.Version()) { + idsToDeleteArr = append(idsToDeleteArr, version.ID()) + } } } - return objInfo, nil + return idsToDeleteArr } -func (n *layer) headLastVersion(ctx context.Context, bkt *BucketInfo, objectName string) (*ObjectInfo, error) { +func (n *layer) headLastVersionIfNotDeleted(ctx context.Context, bkt *BucketInfo, objectName string) (*ObjectInfo, error) { + versions, err := n.headVersions(ctx, bkt, objectName) + if err != nil { + return nil, err + } + + lastVersion := versions.getLast() + if lastVersion == nil { + return nil, apiErrors.GetAPIError(apiErrors.ErrNoSuchKey) + } + return lastVersion, nil +} + +func (n *layer) headVersions(ctx context.Context, bkt *BucketInfo, objectName string) (*objectVersions, error) { ids, err := n.objectSearch(ctx, &findParams{cid: bkt.CID, val: objectName}) if err != nil { return nil, err @@ -255,7 +272,7 @@ func (n *layer) headLastVersion(ctx context.Context, bkt *BucketInfo, objectName return nil, apiErrors.GetAPIError(apiErrors.ErrNoSuchKey) } - infos := make([]*object.Object, 0, len(ids)) + versions := &objectVersions{} for _, id := range ids { meta, err := n.objectHead(ctx, bkt.CID, id) if err != nil { @@ -265,14 +282,15 @@ func (n *layer) headLastVersion(ctx context.Context, bkt *BucketInfo, objectName zap.Error(err)) continue } - infos = append(infos, meta) + if oi := objectInfoFromMeta(bkt, meta, "", ""); oi != nil { + if isSystem(oi) { + continue + } + versions.appendVersion(oi) + } } - sort.Slice(infos, func(i, j int) bool { - return infos[i].CreationEpoch() < infos[j].CreationEpoch() || (infos[i].CreationEpoch() == infos[j].CreationEpoch() && infos[i].ID().String() < infos[j].ID().String()) - }) - - return objectInfoFromMeta(bkt, infos[len(infos)-1], "", ""), nil + return versions, nil } func (n *layer) headVersion(ctx context.Context, bkt *BucketInfo, versionID string) (*ObjectInfo, error) { @@ -378,17 +396,12 @@ func (n *layer) ListObjectsV2(ctx context.Context, p *ListObjectsParamsV2) (*Lis } func (n *layer) listSortedObjectsFromNeoFS(ctx context.Context, p allObjectParams) ([]*ObjectInfo, error) { - var ( - err error - ids []*object.ID - uniqNames = make(map[string]bool) - ) - - if ids, err = n.objectSearch(ctx, &findParams{cid: p.Bucket.CID}); err != nil { + ids, err := n.objectSearch(ctx, &findParams{cid: p.Bucket.CID}) + if err != nil { return nil, err } - objects := make([]*ObjectInfo, 0, len(ids)) + versions := make(map[string]*objectVersions, len(ids)/2) for _, id := range ids { meta, err := n.objectHead(ctx, p.Bucket.CID, id) @@ -397,14 +410,26 @@ func (n *layer) listSortedObjectsFromNeoFS(ctx context.Context, p allObjectParam continue } if oi := objectInfoFromMeta(p.Bucket, meta, p.Prefix, p.Delimiter); oi != nil { - // use only unique dir names - if _, ok := uniqNames[oi.Name]; ok { + if isSystem(oi) { continue } - uniqNames[oi.Name] = oi.isDir + if objVersions, ok := versions[oi.Name]; ok { + objVersions.appendVersion(oi) + versions[oi.Name] = objVersions + } else { + objVersion := &objectVersions{} + objVersion.appendVersion(oi) + versions[oi.Name] = objVersion + } + } + } - objects = append(objects, oi) + objects := make([]*ObjectInfo, 0, len(versions)) + for _, v := range versions { + lastVersion := v.getLast() + if lastVersion != nil { + objects = append(objects, lastVersion) } } @@ -415,6 +440,29 @@ func (n *layer) listSortedObjectsFromNeoFS(ctx context.Context, p allObjectParam return objects, nil } +func getExistedVersions(versions *objectVersions) []string { + var res []string + for _, add := range versions.addList { + if !contains(versions.delList, add) { + res = append(res, add) + } + } + return res +} + +func splitVersions(header string) []string { + if len(header) == 0 { + return nil + } + + return strings.Split(header, ",") +} + +func isSystem(obj *ObjectInfo) bool { + return len(obj.Headers[objectSystemAttributeName]) > 0 || + len(obj.Headers[attrVersionsIgnore]) > 0 +} + func trimAfterObjectName(startAfter string, objects []*ObjectInfo) []*ObjectInfo { if len(objects) != 0 && objects[len(objects)-1].Name <= startAfter { return nil diff --git a/api/layer/util.go b/api/layer/util.go index 6fce6919..a915c5d3 100644 --- a/api/layer/util.go +++ b/api/layer/util.go @@ -9,7 +9,6 @@ import ( "time" cid "github.com/nspcc-dev/neofs-api-go/pkg/container/id" - "github.com/nspcc-dev/neofs-api-go/pkg/object" "github.com/nspcc-dev/neofs-api-go/pkg/owner" "github.com/nspcc-dev/neofs-s3-gw/api" @@ -19,18 +18,19 @@ import ( type ( // ObjectInfo holds S3 object data. ObjectInfo struct { - id *object.ID - isDir bool + id *object.ID + bucketID *cid.ID + isDir bool - Bucket string - bucketID *cid.ID - Name string - Size int64 - ContentType string - Created time.Time - HashSum string - Owner *owner.ID - Headers map[string]string + Bucket string + Name string + Size int64 + ContentType string + Created time.Time + CreationEpoch uint64 + HashSum string + Owner *owner.ID + Headers map[string]string } // ListObjectsInfo contains common fields of data for ListObjectsV1 and ListObjectsV2. @@ -54,19 +54,8 @@ type ( // ObjectVersionInfo stores info about objects versions. ObjectVersionInfo struct { - Object *ObjectInfo - IsLatest bool - VersionID string - CreationEpoch uint64 - } - - // DeletedObjectInfo stores info about deleted versions of objects. - DeletedObjectInfo struct { - Owner *owner.ID - Key string - VersionID string - IsLatest bool - LastModified string + Object *ObjectInfo + IsLatest bool } // ListObjectVersionsInfo stores info and list of objects' versions. @@ -77,7 +66,7 @@ type ( NextKeyMarker string NextVersionIDMarker string Version []*ObjectVersionInfo - DeleteMarker []*DeletedObjectInfo + DeleteMarker []*ObjectVersionInfo VersionIDMarker string } ) @@ -137,29 +126,22 @@ func objectInfoFromMeta(bkt *BucketInfo, meta *object.Object, prefix, delimiter } return &ObjectInfo{ - id: meta.ID(), - isDir: isDir, + id: meta.ID(), + bucketID: bkt.CID, + isDir: isDir, - Bucket: bkt.Name, - bucketID: bkt.CID, - Name: filename, - Created: creation, - ContentType: mimeType, - Headers: userHeaders, - Owner: meta.OwnerID(), - Size: size, - HashSum: meta.PayloadChecksum().String(), + Bucket: bkt.Name, + Name: filename, + Created: creation, + CreationEpoch: meta.CreationEpoch(), + ContentType: mimeType, + Headers: userHeaders, + Owner: meta.OwnerID(), + Size: size, + HashSum: meta.PayloadChecksum().String(), } } -func objectVersionInfoFromMeta(bkt *BucketInfo, meta *object.Object, prefix, delimiter string) *ObjectVersionInfo { - oi := objectInfoFromMeta(bkt, meta, prefix, delimiter) - if oi == nil { - return nil - } - return &ObjectVersionInfo{Object: oi, VersionID: meta.ID().String(), CreationEpoch: meta.CreationEpoch()} -} - func filenameFromObject(o *object.Object) string { var name = o.ID().String() for _, attr := range o.Attributes() { @@ -179,6 +161,9 @@ func NameFromString(name string) (string, string) { // ID returns object ID from ObjectInfo. func (o *ObjectInfo) ID() *object.ID { return o.id } +// Version returns object version from ObjectInfo. +func (o *ObjectInfo) Version() string { return o.id.String() } + // CID returns bucket ID from ObjectInfo. func (o *ObjectInfo) CID() *cid.ID { return o.bucketID } diff --git a/api/layer/versioning_test.go b/api/layer/versioning_test.go new file mode 100644 index 00000000..31ec3e04 --- /dev/null +++ b/api/layer/versioning_test.go @@ -0,0 +1,608 @@ +package layer + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/sha256" + "fmt" + "io" + "strings" + "testing" + + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "github.com/nspcc-dev/neofs-api-go/pkg/acl/eacl" + "github.com/nspcc-dev/neofs-api-go/pkg/client" + "github.com/nspcc-dev/neofs-api-go/pkg/container" + cid "github.com/nspcc-dev/neofs-api-go/pkg/container/id" + "github.com/nspcc-dev/neofs-api-go/pkg/object" + "github.com/nspcc-dev/neofs-api-go/pkg/owner" + "github.com/nspcc-dev/neofs-api-go/pkg/session" + "github.com/nspcc-dev/neofs-api-go/pkg/token" + "github.com/nspcc-dev/neofs-s3-gw/api" + "github.com/nspcc-dev/neofs-s3-gw/creds/accessbox" + "github.com/nspcc-dev/neofs-sdk-go/pkg/logger" + "github.com/nspcc-dev/neofs-sdk-go/pkg/pool" + "github.com/stretchr/testify/require" +) + +type testPool struct { + objects map[string]*object.Object + containers map[string]*container.Container + currentEpoch uint64 +} + +func newTestPool() *testPool { + return &testPool{ + objects: make(map[string]*object.Object), + containers: make(map[string]*container.Container), + } +} + +func (t *testPool) PutObject(ctx context.Context, params *client.PutObjectParams, option ...client.CallOption) (*object.ID, error) { + b := make([]byte, 32) + if _, err := io.ReadFull(rand.Reader, b); err != nil { + return nil, err + } + + oid := object.NewID() + oid.SetSHA256(sha256.Sum256(b)) + + raw := object.NewRawFrom(params.Object()) + raw.SetID(oid) + raw.SetCreationEpoch(t.currentEpoch) + t.currentEpoch++ + + if params.PayloadReader() != nil { + all, err := io.ReadAll(params.PayloadReader()) + if err != nil { + return nil, err + } + raw.SetPayload(all) + } + + addr := object.NewAddress() + addr.SetObjectID(raw.ID()) + addr.SetContainerID(raw.ContainerID()) + + t.objects[addr.String()] = raw.Object() + return raw.ID(), nil +} + +func (t *testPool) DeleteObject(ctx context.Context, params *client.DeleteObjectParams, option ...client.CallOption) error { + delete(t.objects, params.Address().String()) + return nil +} + +func (t *testPool) GetObject(ctx context.Context, params *client.GetObjectParams, option ...client.CallOption) (*object.Object, error) { + if obj, ok := t.objects[params.Address().String()]; ok { + if params.PayloadWriter() != nil { + _, err := params.PayloadWriter().Write(obj.Payload()) + if err != nil { + return nil, err + } + } + return obj, nil + } + + return nil, fmt.Errorf("object not found " + params.Address().String()) +} + +func (t *testPool) GetObjectHeader(ctx context.Context, params *client.ObjectHeaderParams, option ...client.CallOption) (*object.Object, error) { + p := new(client.GetObjectParams).WithAddress(params.Address()) + return t.GetObject(ctx, p) +} + +func (t *testPool) ObjectPayloadRangeData(ctx context.Context, params *client.RangeDataParams, option ...client.CallOption) ([]byte, error) { + panic("implement me") +} + +func (t *testPool) ObjectPayloadRangeSHA256(ctx context.Context, params *client.RangeChecksumParams, option ...client.CallOption) ([][32]byte, error) { + panic("implement me") +} + +func (t *testPool) ObjectPayloadRangeTZ(ctx context.Context, params *client.RangeChecksumParams, option ...client.CallOption) ([][64]byte, error) { + panic("implement me") +} + +func (t *testPool) SearchObject(ctx context.Context, params *client.SearchObjectParams, option ...client.CallOption) ([]*object.ID, error) { + cidStr := params.ContainerID().String() + + var res []*object.ID + + if len(params.SearchFilters()) == 1 { + for k, v := range t.objects { + if strings.Contains(k, cidStr) { + res = append(res, v.ID()) + } + } + return res, nil + } + + filter := params.SearchFilters()[1] + if len(params.SearchFilters()) != 2 || filter.Operation() != object.MatchStringEqual || + (filter.Header() != object.AttributeFileName && filter.Header() != objectSystemAttributeName) { + return nil, fmt.Errorf("usupported filters") + } + + for k, v := range t.objects { + if strings.Contains(k, cidStr) && isMatched(v.Attributes(), filter) { + res = append(res, v.ID()) + } + } + + return res, nil +} + +func isMatched(attributes []*object.Attribute, filter object.SearchFilter) bool { + for _, attr := range attributes { + if attr.Key() == filter.Header() && attr.Value() == filter.Value() { + return true + } + } + + return false +} + +func (t *testPool) PutContainer(ctx context.Context, container *container.Container, option ...client.CallOption) (*cid.ID, error) { + b := make([]byte, 32) + if _, err := io.ReadFull(rand.Reader, b); err != nil { + return nil, err + } + + id := cid.New() + id.SetSHA256(sha256.Sum256(b)) + t.containers[id.String()] = container + + return id, nil +} + +func (t *testPool) GetContainer(ctx context.Context, id *cid.ID, option ...client.CallOption) (*container.Container, error) { + for k, v := range t.containers { + if k == id.String() { + return v, nil + } + } + + return nil, fmt.Errorf("container not found " + id.String()) +} + +func (t *testPool) ListContainers(ctx context.Context, id *owner.ID, option ...client.CallOption) ([]*cid.ID, error) { + var res []*cid.ID + for k := range t.containers { + cID := cid.New() + if err := cID.Parse(k); err != nil { + return nil, err + } + res = append(res, cID) + } + + return res, nil +} + +func (t *testPool) DeleteContainer(ctx context.Context, id *cid.ID, option ...client.CallOption) error { + delete(t.containers, id.String()) + return nil +} + +func (t *testPool) GetEACL(ctx context.Context, id *cid.ID, option ...client.CallOption) (*client.EACLWithSignature, error) { + panic("implement me") +} + +func (t *testPool) SetEACL(ctx context.Context, table *eacl.Table, option ...client.CallOption) error { + panic("implement me") +} + +func (t *testPool) AnnounceContainerUsedSpace(ctx context.Context, announcements []container.UsedSpaceAnnouncement, option ...client.CallOption) error { + panic("implement me") +} + +func (t *testPool) Connection() (client.Client, *session.Token, error) { + panic("implement me") +} + +func (t *testPool) OwnerID() *owner.ID { + return nil +} + +func (t *testPool) WaitForContainerPresence(ctx context.Context, id *cid.ID, params *pool.ContainerPollingParams) error { + return nil +} + +func (tc *testContext) putObject(content []byte) *ObjectInfo { + objInfo, err := tc.layer.PutObject(tc.ctx, &PutObjectParams{ + Bucket: tc.bkt, + Object: tc.obj, + Size: int64(len(content)), + Reader: bytes.NewReader(content), + Header: make(map[string]string), + }) + require.NoError(tc.t, err) + + return objInfo +} + +func (tc *testContext) getObject(objectName, versionID string, needError bool) (*ObjectInfo, []byte) { + objInfo, err := tc.layer.GetObjectInfo(tc.ctx, &HeadObjectParams{ + Bucket: tc.bkt, + Object: objectName, + VersionID: versionID, + }) + if needError { + require.Error(tc.t, err) + return nil, nil + } + require.NoError(tc.t, err) + + content := bytes.NewBuffer(nil) + err = tc.layer.GetObject(tc.ctx, &GetObjectParams{ + ObjectInfo: objInfo, + Writer: content, + VersionID: versionID, + }) + require.NoError(tc.t, err) + + return objInfo, content.Bytes() +} + +func (tc *testContext) deleteObject(objectName, versionID string) { + errs := tc.layer.DeleteObjects(tc.ctx, tc.bkt, []*VersionedObject{ + {Name: objectName, VersionID: versionID}, + }) + for _, err := range errs { + require.NoError(tc.t, err) + } +} + +func (tc *testContext) listObjectsV1() []*ObjectInfo { + res, err := tc.layer.ListObjectsV1(tc.ctx, &ListObjectsParamsV1{ + ListObjectsParamsCommon: ListObjectsParamsCommon{ + Bucket: tc.bkt, + MaxKeys: 1000, + }, + }) + require.NoError(tc.t, err) + return res.Objects +} + +func (tc *testContext) listObjectsV2() []*ObjectInfo { + res, err := tc.layer.ListObjectsV2(tc.ctx, &ListObjectsParamsV2{ + ListObjectsParamsCommon: ListObjectsParamsCommon{ + Bucket: tc.bkt, + MaxKeys: 1000, + }, + }) + require.NoError(tc.t, err) + return res.Objects +} + +func (tc *testContext) listVersions() *ListObjectVersionsInfo { + res, err := tc.layer.ListObjectVersions(tc.ctx, &ListObjectVersionsParams{ + Bucket: tc.bkt, + MaxKeys: 1000, + }) + require.NoError(tc.t, err) + return res +} + +func (tc *testContext) checkListObjects(ids ...*object.ID) { + objs := tc.listObjectsV1() + require.Equal(tc.t, len(ids), len(objs)) + for _, id := range ids { + require.Contains(tc.t, ids, id) + } + + objs = tc.listObjectsV2() + require.Equal(tc.t, len(ids), len(objs)) + for _, id := range ids { + require.Contains(tc.t, ids, id) + } +} + +type testContext struct { + t *testing.T + ctx context.Context + layer Client + bkt string + bktID *cid.ID + obj string + testPool *testPool +} + +func prepareContext(t *testing.T) *testContext { + key, err := keys.NewPrivateKey() + require.NoError(t, err) + + ctx := context.WithValue(context.Background(), api.BoxData, &accessbox.Box{ + Gate: &accessbox.GateData{ + BearerToken: token.NewBearerToken(), + GateKey: key.PublicKey(), + }, + }) + l, err := logger.New(logger.WithTraceLevel("panic")) + require.NoError(t, err) + tp := newTestPool() + + bktName := "testbucket1" + cnr := container.New(container.WithAttribute(container.AttributeName, bktName)) + bktID, err := tp.PutContainer(ctx, cnr) + require.NoError(t, err) + + return &testContext{ + ctx: ctx, + layer: NewLayer(l, tp), + bkt: bktName, + bktID: bktID, + obj: "obj1", + t: t, + testPool: tp, + } +} + +func TestSimpleVersioning(t *testing.T) { + tc := prepareContext(t) + _, err := tc.layer.PutBucketVersioning(tc.ctx, &PutVersioningParams{ + Bucket: tc.bkt, + Settings: &BucketSettings{VersioningEnabled: true}, + }) + require.NoError(t, err) + + obj1Content1 := []byte("content obj1 v1") + obj1v1 := tc.putObject(obj1Content1) + + obj1Content2 := []byte("content obj1 v2") + obj1v2 := tc.putObject(obj1Content2) + + objv2, buffer2 := tc.getObject(tc.obj, "", false) + require.Equal(t, obj1Content2, buffer2) + require.Contains(t, objv2.Headers[versionsAddAttr], obj1v1.ID().String()) + + _, buffer1 := tc.getObject(tc.obj, obj1v1.ID().String(), false) + require.Equal(t, obj1Content1, buffer1) + + tc.checkListObjects(obj1v2.ID()) +} + +func TestSimpleNoVersioning(t *testing.T) { + tc := prepareContext(t) + + obj1Content1 := []byte("content obj1 v1") + obj1v1 := tc.putObject(obj1Content1) + + obj1Content2 := []byte("content obj1 v2") + obj1v2 := tc.putObject(obj1Content2) + + objv2, buffer2 := tc.getObject(tc.obj, "", false) + require.Equal(t, obj1Content2, buffer2) + require.Contains(t, objv2.Headers[versionsDelAttr], obj1v1.ID().String()) + + tc.getObject(tc.obj, obj1v1.ID().String(), true) + tc.checkListObjects(obj1v2.ID()) +} + +func TestVersioningDeleteObject(t *testing.T) { + tc := prepareContext(t) + _, err := tc.layer.PutBucketVersioning(tc.ctx, &PutVersioningParams{ + Bucket: tc.bkt, + Settings: &BucketSettings{VersioningEnabled: true}, + }) + require.NoError(t, err) + + tc.putObject([]byte("content obj1 v1")) + tc.putObject([]byte("content obj1 v2")) + + tc.deleteObject(tc.obj, "") + tc.getObject(tc.obj, "", true) + + tc.checkListObjects() +} + +func TestVersioningDeleteSpecificObjectVersion(t *testing.T) { + tc := prepareContext(t) + _, err := tc.layer.PutBucketVersioning(tc.ctx, &PutVersioningParams{ + Bucket: tc.bkt, + Settings: &BucketSettings{VersioningEnabled: true}, + }) + require.NoError(t, err) + + tc.putObject([]byte("content obj1 v1")) + objV2Info := tc.putObject([]byte("content obj1 v2")) + objV3Content := []byte("content obj1 v3") + objV3Info := tc.putObject(objV3Content) + + tc.deleteObject(tc.obj, objV2Info.Version()) + tc.getObject(tc.obj, objV2Info.Version(), true) + + _, buffer3 := tc.getObject(tc.obj, "", false) + require.Equal(t, objV3Content, buffer3) + + tc.deleteObject(tc.obj, "") + tc.getObject(tc.obj, "", true) + + for _, ver := range tc.listVersions().DeleteMarker { + if ver.IsLatest { + tc.deleteObject(tc.obj, ver.Object.Version()) + } + } + + resInfo, buffer := tc.getObject(tc.obj, "", false) + require.Equal(t, objV3Content, buffer) + require.Equal(t, objV3Info.Version(), resInfo.Version()) +} + +func TestNoVersioningDeleteObject(t *testing.T) { + tc := prepareContext(t) + + tc.putObject([]byte("content obj1 v1")) + tc.putObject([]byte("content obj1 v2")) + + tc.deleteObject(tc.obj, "") + tc.getObject(tc.obj, "", true) + tc.checkListObjects() +} + +func TestGetLastVersion(t *testing.T) { + obj1 := getTestObjectInfo(1, getOID(1), "", "", "") + obj1V2 := getTestObjectInfo(1, getOID(2), "", "", "") + obj2 := getTestObjectInfo(2, getOID(2), obj1.Version(), "", "") + obj3 := getTestObjectInfo(3, getOID(3), joinVers(obj1, obj2), "", "*") + obj4 := getTestObjectInfo(4, getOID(4), joinVers(obj1, obj2), obj2.Version(), obj2.Version()) + obj5 := getTestObjectInfo(5, getOID(5), obj1.Version(), obj1.Version(), obj1.Version()) + obj6 := getTestObjectInfo(6, getOID(6), joinVers(obj1, obj2, obj3), obj3.Version(), obj3.Version()) + + for _, tc := range []struct { + versions *objectVersions + expected *ObjectInfo + }{ + { + versions: &objectVersions{}, + expected: nil, + }, + { + versions: &objectVersions{ + objects: []*ObjectInfo{obj2, obj1}, + addList: []string{obj1.Version(), obj2.Version()}, + }, + expected: obj2, + }, + { + versions: &objectVersions{ + objects: []*ObjectInfo{obj2, obj1, obj3}, + addList: []string{obj1.Version(), obj2.Version(), obj3.Version()}, + }, + expected: nil, + }, + { + versions: &objectVersions{ + objects: []*ObjectInfo{obj2, obj1, obj4}, + addList: []string{obj1.Version(), obj2.Version(), obj4.Version()}, + delList: []string{obj2.Version()}, + }, + expected: obj1, + }, + { + versions: &objectVersions{ + objects: []*ObjectInfo{obj1, obj5}, + addList: []string{obj1.Version(), obj5.Version()}, + delList: []string{obj1.Version()}, + }, + expected: nil, + }, + { + versions: &objectVersions{ + objects: []*ObjectInfo{obj5}, + }, + expected: nil, + }, + { + versions: &objectVersions{ + objects: []*ObjectInfo{obj1, obj2, obj3, obj6}, + addList: []string{obj1.Version(), obj2.Version(), obj3.Version(), obj6.Version()}, + delList: []string{obj3.Version()}, + }, + expected: obj2, + }, + { + versions: &objectVersions{ + objects: []*ObjectInfo{obj1, obj1V2}, + addList: []string{obj1.Version(), obj1V2.Version()}, + }, + // creation epochs are equal + // obj1 version/oid > obj1_1 version/oid + expected: obj1, + }, + } { + actualObjInfo := tc.versions.getLast() + require.Equal(t, tc.expected, actualObjInfo) + } +} + +func TestAppendVersions(t *testing.T) { + obj1 := getTestObjectInfo(1, getOID(1), "", "", "") + obj2 := getTestObjectInfo(2, getOID(2), obj1.Version(), "", "") + obj3 := getTestObjectInfo(3, getOID(3), joinVers(obj1, obj2), "", "*") + obj4 := getTestObjectInfo(4, getOID(4), joinVers(obj1, obj2), obj2.Version(), obj2.Version()) + + for _, tc := range []struct { + versions *objectVersions + objectToAdd *ObjectInfo + expectedVersions *objectVersions + }{ + { + versions: &objectVersions{}, + objectToAdd: obj1, + expectedVersions: &objectVersions{ + objects: []*ObjectInfo{obj1}, + addList: []string{obj1.Version()}, + }, + }, + { + versions: &objectVersions{objects: []*ObjectInfo{obj1}}, + objectToAdd: obj2, + expectedVersions: &objectVersions{ + objects: []*ObjectInfo{obj1, obj2}, + addList: []string{obj1.Version(), obj2.Version()}, + }, + }, + { + versions: &objectVersions{objects: []*ObjectInfo{obj1, obj2}}, + objectToAdd: obj3, + expectedVersions: &objectVersions{ + objects: []*ObjectInfo{obj1, obj2, obj3}, + addList: []string{obj1.Version(), obj2.Version(), obj3.Version()}, + }, + }, + { + versions: &objectVersions{objects: []*ObjectInfo{obj1, obj2}}, + objectToAdd: obj4, + expectedVersions: &objectVersions{ + objects: []*ObjectInfo{obj1, obj2, obj4}, + addList: []string{obj1.Version(), obj2.Version(), obj4.Version()}, + delList: []string{obj2.Version()}, + }, + }, + } { + tc.versions.appendVersion(tc.objectToAdd) + require.Equal(t, tc.expectedVersions, tc.versions) + } +} + +func joinVers(objs ...*ObjectInfo) string { + if len(objs) == 0 { + return "" + } + + var versions []string + for _, obj := range objs { + versions = append(versions, obj.Version()) + } + + return strings.Join(versions, ",") +} + +func getOID(id byte) *object.ID { + b := make([]byte, 32) + b[0] = id + oid := object.NewID() + oid.SetSHA256(sha256.Sum256(b)) + return oid +} + +func getTestObjectInfo(epoch uint64, oid *object.ID, addAttr, delAttr, delMarkAttr string) *ObjectInfo { + headers := make(map[string]string) + if addAttr != "" { + headers[versionsAddAttr] = addAttr + } + if delAttr != "" { + headers[versionsDelAttr] = delAttr + } + if delMarkAttr != "" { + headers[versionsDeleteMarkAttr] = delMarkAttr + } + + return &ObjectInfo{ + id: oid, + CreationEpoch: epoch, + Headers: headers, + } +} From f6c51cc9ee4250b653da2319f3005936516a06c6 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Mon, 16 Aug 2021 12:31:50 +0300 Subject: [PATCH 08/10] [#122] Update listObjectVerions Signed-off-by: Denis Kirillov --- api/layer/layer.go | 86 +++++++++++++++++++++++++++++---------------- api/layer/object.go | 10 ++---- 2 files changed, 57 insertions(+), 39 deletions(-) diff --git a/api/layer/layer.go b/api/layer/layer.go index fbd915fe..c454ac26 100644 --- a/api/layer/layer.go +++ b/api/layer/layer.go @@ -137,6 +137,7 @@ type ( } objectVersions struct { + name string objects []*ObjectInfo addList []string delList []string @@ -177,6 +178,10 @@ type ( } ) +func newObjectVersions(name string) *objectVersions { + return &objectVersions{name: name} +} + func (v *objectVersions) appendVersion(oi *ObjectInfo) { addVers := append(splitVersions(oi.Headers[versionsAddAttr]), oi.Version()) delVers := splitVersions(oi.Headers[versionsDelAttr]) @@ -214,7 +219,7 @@ func (v *objectVersions) getLast() *ObjectInfo { if delMarkHeader == "" { return v.objects[i] } - if delMarkHeader == "*" { + if delMarkHeader == delMarkFullObject { return nil } } @@ -223,6 +228,29 @@ func (v *objectVersions) getLast() *ObjectInfo { return nil } +func (v *objectVersions) getFiltered() []*ObjectVersionInfo { + if len(v.objects) == 0 { + return nil + } + + v.sort() + existedVersions := getExistedVersions(v) + res := make([]*ObjectVersionInfo, 0, len(v.objects)) + + for _, version := range v.objects { + delMark := version.Headers[versionsDeleteMarkAttr] + if contains(existedVersions, version.Version()) && (delMark == delMarkFullObject || delMark == "") { + res = append(res, &ObjectVersionInfo{Object: version}) + } + } + + if len(res) > 0 { + res[len(res)-1].IsLatest = true + } + + return res +} + func (v *objectVersions) getAddHeader() string { return strings.Join(v.addList, ",") } @@ -237,6 +265,10 @@ const ( objectSystemAttributeName = "S3-System-name" attrVersionsIgnore = "S3-Versions-ignore" attrSettingsVersioningEnabled = "S3-Settings-Versioning-enabled" + versionsDelAttr = "S3-Versions-del" + versionsAddAttr = "S3-Versions-add" + versionsDeleteMarkAttr = "S3-Versions-delete-mark" + delMarkFullObject = "*" ) func (t *VersionedObject) String() string { @@ -490,7 +522,7 @@ func (n *layer) deleteObject(ctx context.Context, bkt *BucketInfo, obj *Versione p.Header[versionsDelAttr] = obj.VersionID } else { - p.Header[versionsDeleteMarkAttr] = "*" + p.Header[versionsDeleteMarkAttr] = delMarkFullObject } if _, err = n.objectPut(ctx, bkt, p); err != nil { return &errors.DeleteError{Err: err, Object: obj.String()} @@ -606,26 +638,29 @@ func (n *layer) ListObjectVersions(ctx context.Context, p *ListObjectVersionsPar objVersions.appendVersion(oi) versions[oi.Name] = objVersions } else { - objVersion := &objectVersions{} + objVersion := newObjectVersions(oi.Name) objVersion.appendVersion(oi) versions[oi.Name] = objVersion } } } - for _, v := range versions { - existed, deleted := triageVersions(v) - res.Version = append(res.Version, existed...) - res.DeleteMarker = append(res.DeleteMarker, deleted...) + sortedNames := make([]string, 0, len(versions)) + for k := range versions { + sortedNames = append(sortedNames, k) + } + sort.Strings(sortedNames) + + objects := make([]*ObjectVersionInfo, 0, p.MaxKeys) + for _, name := range sortedNames { + objects = append(objects, versions[name].getFiltered()...) + if len(objects) > p.MaxKeys { + objects = objects[:p.MaxKeys] + break + } } - sort.Slice(res.Version, func(i, j int) bool { - return res.Version[i].Object.Name < res.Version[j].Object.Name - }) - sort.Slice(res.DeleteMarker, func(i, j int) bool { - return res.DeleteMarker[i].Object.Name < res.DeleteMarker[j].Object.Name - }) - + res.Version, res.DeleteMarker = triageVersions(objects) return &res, nil } @@ -635,30 +670,19 @@ func sortVersions(versions []*ObjectInfo) { }) } -func triageVersions(objectVersions *objectVersions) ([]*ObjectVersionInfo, []*ObjectVersionInfo) { - if objectVersions == nil || len(objectVersions.objects) == 0 { +func triageVersions(objVersions []*ObjectVersionInfo) ([]*ObjectVersionInfo, []*ObjectVersionInfo) { + if len(objVersions) == 0 { return nil, nil } - sortVersions(objectVersions.objects) - var resVersion []*ObjectVersionInfo var resDelMarkVersions []*ObjectVersionInfo - isLatest := true - for i := len(objectVersions.objects) - 1; i >= 0; i-- { - version := objectVersions.objects[i] - if contains(objectVersions.delList, version.Version()) { - continue - } - - ovi := &ObjectVersionInfo{Object: version, IsLatest: isLatest} - isLatest = false - - if len(version.Headers[versionsDeleteMarkAttr]) == 0 { - resVersion = append(resVersion, ovi) + for _, version := range objVersions { + if version.Object.Headers[versionsDeleteMarkAttr] == delMarkFullObject { + resDelMarkVersions = append(resDelMarkVersions, version) } else { - resDelMarkVersions = append(resDelMarkVersions, ovi) + resVersion = append(resVersion, version) } } diff --git a/api/layer/object.go b/api/layer/object.go index b77c3589..b418af03 100644 --- a/api/layer/object.go +++ b/api/layer/object.go @@ -64,12 +64,6 @@ type ( } ) -const ( - versionsDelAttr = "S3-Versions-del" - versionsAddAttr = "S3-Versions-add" - versionsDeleteMarkAttr = "S3-Versions-delete-mark" -) - // objectSearch returns all available objects by search params. func (n *layer) objectSearch(ctx context.Context, p *findParams) ([]*object.ID, error) { var opts object.SearchFilters @@ -272,7 +266,7 @@ func (n *layer) headVersions(ctx context.Context, bkt *BucketInfo, objectName st return nil, apiErrors.GetAPIError(apiErrors.ErrNoSuchKey) } - versions := &objectVersions{} + versions := newObjectVersions(objectName) for _, id := range ids { meta, err := n.objectHead(ctx, bkt.CID, id) if err != nil { @@ -418,7 +412,7 @@ func (n *layer) listSortedObjectsFromNeoFS(ctx context.Context, p allObjectParam objVersions.appendVersion(oi) versions[oi.Name] = objVersions } else { - objVersion := &objectVersions{} + objVersion := newObjectVersions(oi.Name) objVersion.appendVersion(oi) versions[oi.Name] = objVersion } From 11558124cd75239a7e30133f0dfd5ecac066e316 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Wed, 18 Aug 2021 16:48:58 +0300 Subject: [PATCH 09/10] [#122] Add versioning cache Signed-off-by: Denis Kirillov --- api/cache/bucket.go | 64 ++++++++++++ api/cache/head_cache.go | 55 ++++++++++ api/cache/object_cache.go | 6 +- api/cache/object_cache_test.go | 16 +-- api/cache/system.go | 56 ++++++++++ api/handler/head.go | 26 +++-- api/handler/object_list.go | 2 +- api/layer/container.go | 27 +++-- api/layer/detector.go | 58 ++++++++--- api/layer/layer.go | 182 +++++++++++++++++---------------- api/layer/object.go | 171 +++++++++++++++++++++---------- api/layer/object_list_cache.go | 7 +- api/layer/util.go | 15 ++- api/layer/util_test.go | 7 +- api/layer/versioning_test.go | 14 +-- 15 files changed, 503 insertions(+), 203 deletions(-) create mode 100644 api/cache/bucket.go create mode 100644 api/cache/head_cache.go create mode 100644 api/cache/system.go diff --git a/api/cache/bucket.go b/api/cache/bucket.go new file mode 100644 index 00000000..c6896bdb --- /dev/null +++ b/api/cache/bucket.go @@ -0,0 +1,64 @@ +package cache + +import ( + "time" + + "github.com/bluele/gcache" + cid "github.com/nspcc-dev/neofs-api-go/pkg/container/id" + "github.com/nspcc-dev/neofs-api-go/pkg/owner" +) + +type ( + // BucketCache provides interface for lru cache for objects. + BucketCache interface { + Get(key string) *BucketInfo + Put(bkt *BucketInfo) error + Delete(key string) bool + } + + // BucketInfo stores basic bucket data. + BucketInfo struct { + Name string + CID *cid.ID + Owner *owner.ID + Created time.Time + } + + // GetBucketCache contains cache with objects and lifetime of cache entries. + GetBucketCache struct { + cache gcache.Cache + lifetime time.Duration + } +) + +// NewBucketCache creates an object of BucketCache. +func NewBucketCache(cacheSize int, lifetime time.Duration) *GetBucketCache { + gc := gcache.New(cacheSize).LRU().Build() + + return &GetBucketCache{cache: gc, lifetime: lifetime} +} + +// Get returns cached object. +func (o *GetBucketCache) Get(key string) *BucketInfo { + entry, err := o.cache.Get(key) + if err != nil { + return nil + } + + result, ok := entry.(*BucketInfo) + if !ok { + return nil + } + + return result +} + +// Put puts an object to cache. +func (o *GetBucketCache) Put(bkt *BucketInfo) error { + return o.cache.SetWithExpire(bkt.Name, bkt, o.lifetime) +} + +// Delete deletes an object from cache. +func (o *GetBucketCache) Delete(key string) bool { + return o.cache.Remove(key) +} diff --git a/api/cache/head_cache.go b/api/cache/head_cache.go new file mode 100644 index 00000000..08ac3d13 --- /dev/null +++ b/api/cache/head_cache.go @@ -0,0 +1,55 @@ +package cache + +import ( + "time" + + "github.com/bluele/gcache" + "github.com/nspcc-dev/neofs-api-go/pkg/object" +) + +// HeadObjectsCache provides interface for lru cache for objects. +type HeadObjectsCache interface { + Get(key string) *object.Address + Put(key string, address *object.Address) error + Delete(key string) bool +} + +type ( + // HeadObjectCache contains cache with objects and lifetime of cache entries. + HeadObjectCache struct { + cache gcache.Cache + lifetime time.Duration + } +) + +// NewHeadObject creates an object of ObjectHeadersCache. +func NewHeadObject(cacheSize int, lifetime time.Duration) *HeadObjectCache { + gc := gcache.New(cacheSize).LRU().Build() + + return &HeadObjectCache{cache: gc, lifetime: lifetime} +} + +// Get returns cached object. +func (o *HeadObjectCache) Get(key string) *object.Address { + entry, err := o.cache.Get(key) + if err != nil { + return nil + } + + result, ok := entry.(*object.Address) + if !ok { + return nil + } + + return result +} + +// Put puts an object to cache. +func (o *HeadObjectCache) Put(key string, address *object.Address) error { + return o.cache.SetWithExpire(key, address, o.lifetime) +} + +// Delete deletes an object from cache. +func (o *HeadObjectCache) Delete(key string) bool { + return o.cache.Remove(key) +} diff --git a/api/cache/object_cache.go b/api/cache/object_cache.go index f7bcc244..708fb696 100644 --- a/api/cache/object_cache.go +++ b/api/cache/object_cache.go @@ -10,7 +10,7 @@ import ( // ObjectsCache provides interface for lru cache for objects. type ObjectsCache interface { Get(address *object.Address) *object.Object - Put(address *object.Address, obj object.Object) error + Put(obj object.Object) error Delete(address *object.Address) bool } @@ -52,8 +52,8 @@ func (o *ObjectHeadersCache) Get(address *object.Address) *object.Object { } // Put puts an object to cache. -func (o *ObjectHeadersCache) Put(address *object.Address, obj object.Object) error { - return o.cache.SetWithExpire(address.String(), obj, o.lifetime) +func (o *ObjectHeadersCache) Put(obj object.Object) error { + return o.cache.SetWithExpire(obj.ContainerID().String()+"/"+obj.ID().String(), obj, o.lifetime) } // Delete deletes an object from cache. diff --git a/api/cache/object_cache_test.go b/api/cache/object_cache_test.go index b9ee9294..e307004a 100644 --- a/api/cache/object_cache_test.go +++ b/api/cache/object_cache_test.go @@ -4,6 +4,8 @@ import ( "testing" "time" + "github.com/nspcc-dev/neofs-api-go/pkg/object" + objecttest "github.com/nspcc-dev/neofs-api-go/pkg/object/test" "github.com/stretchr/testify/require" ) @@ -14,23 +16,23 @@ const ( ) func TestCache(t *testing.T) { - var ( - address = objecttest.Address() - object = objecttest.Object() - ) + obj := objecttest.Object() + address := object.NewAddress() + address.SetContainerID(obj.ContainerID()) + address.SetObjectID(obj.ID()) t.Run("check get", func(t *testing.T) { cache := New(cachesize, lifetime) - err := cache.Put(address, *object) + err := cache.Put(*obj) require.NoError(t, err) actual := cache.Get(address) - require.Equal(t, object, actual) + require.Equal(t, obj, actual) }) t.Run("check delete", func(t *testing.T) { cache := New(cachesize, lifetime) - err := cache.Put(address, *object) + err := cache.Put(*obj) require.NoError(t, err) cache.Delete(address) diff --git a/api/cache/system.go b/api/cache/system.go new file mode 100644 index 00000000..d8f18b00 --- /dev/null +++ b/api/cache/system.go @@ -0,0 +1,56 @@ +package cache + +import ( + "time" + + "github.com/nspcc-dev/neofs-api-go/pkg/object" + + "github.com/bluele/gcache" +) + +type ( + // SystemCache provides interface for lru cache for objects. + SystemCache interface { + Get(key string) *object.Object + Put(key string, obj *object.Object) error + Delete(key string) bool + } + + // systemCache contains cache with objects and lifetime of cache entries. + systemCache struct { + cache gcache.Cache + lifetime time.Duration + } +) + +// NewSystemCache creates an object of SystemCache. +func NewSystemCache(cacheSize int, lifetime time.Duration) SystemCache { + gc := gcache.New(cacheSize).LRU().Build() + + return &systemCache{cache: gc, lifetime: lifetime} +} + +// Get returns cached object. +func (o *systemCache) Get(key string) *object.Object { + entry, err := o.cache.Get(key) + if err != nil { + return nil + } + + result, ok := entry.(*object.Object) + if !ok { + return nil + } + + return result +} + +// Put puts an object to cache. +func (o *systemCache) Put(key string, obj *object.Object) error { + return o.cache.SetWithExpire(key, obj, o.lifetime) +} + +// Delete deletes an object from cache. +func (o *systemCache) Delete(key string) bool { + return o.cache.Remove(key) +} diff --git a/api/handler/head.go b/api/handler/head.go index f6f1a274..1f0a3031 100644 --- a/api/handler/head.go +++ b/api/handler/head.go @@ -46,18 +46,22 @@ func (h *handler) HeadObjectHandler(w http.ResponseWriter, r *http.Request) { h.logAndSendError(w, "could not fetch object info", reqInfo, err) return } - buffer := bytes.NewBuffer(make([]byte, 0, sizeToDetectType)) - getParams := &layer.GetObjectParams{ - ObjectInfo: inf, - Writer: buffer, - Range: getRangeToDetectContentType(inf.Size), - VersionID: reqInfo.URL.Query().Get("versionId"), + + if len(inf.ContentType) == 0 { + buffer := bytes.NewBuffer(make([]byte, 0, sizeToDetectType)) + getParams := &layer.GetObjectParams{ + ObjectInfo: inf, + Writer: buffer, + Range: getRangeToDetectContentType(inf.Size), + VersionID: reqInfo.URL.Query().Get("versionId"), + } + if err = h.obj.GetObject(r.Context(), getParams); err != nil { + h.logAndSendError(w, "could not get object", reqInfo, err, zap.Stringer("oid", inf.ID())) + return + } + inf.ContentType = http.DetectContentType(buffer.Bytes()) } - if err = h.obj.GetObject(r.Context(), getParams); err != nil { - h.logAndSendError(w, "could not get object", reqInfo, err, zap.Stringer("oid", inf.ID())) - return - } - inf.ContentType = http.DetectContentType(buffer.Bytes()) + writeHeaders(w.Header(), inf) w.WriteHeader(http.StatusOK) } diff --git a/api/handler/object_list.go b/api/handler/object_list.go index d0233e98..293e6bb2 100644 --- a/api/handler/object_list.go +++ b/api/handler/object_list.go @@ -263,7 +263,7 @@ func encodeListObjectVersionsToResponse(info *layer.ListObjectVersionsInfo, buck } for _, prefix := range info.CommonPrefixes { - res.CommonPrefixes = append(res.CommonPrefixes, CommonPrefix{Prefix: *prefix}) + res.CommonPrefixes = append(res.CommonPrefixes, CommonPrefix{Prefix: prefix}) } for _, ver := range info.Version { diff --git a/api/layer/container.go b/api/layer/container.go index 3bf7dc0e..1e7b3b56 100644 --- a/api/layer/container.go +++ b/api/layer/container.go @@ -11,37 +11,29 @@ import ( "github.com/nspcc-dev/neofs-api-go/pkg/acl/eacl" "github.com/nspcc-dev/neofs-api-go/pkg/container" cid "github.com/nspcc-dev/neofs-api-go/pkg/container/id" - "github.com/nspcc-dev/neofs-api-go/pkg/owner" "github.com/nspcc-dev/neofs-s3-gw/api" + "github.com/nspcc-dev/neofs-s3-gw/api/cache" "github.com/nspcc-dev/neofs-s3-gw/api/errors" "github.com/nspcc-dev/neofs-sdk-go/pkg/pool" "go.uber.org/zap" ) type ( - // BucketInfo stores basic bucket data. - BucketInfo struct { - Name string - CID *cid.ID - Owner *owner.ID - Created time.Time - BasicACL uint32 - } // BucketACL extends BucketInfo by eacl.Table. BucketACL struct { - Info *BucketInfo + Info *cache.BucketInfo EACL *eacl.Table } ) -func (n *layer) containerInfo(ctx context.Context, cid *cid.ID) (*BucketInfo, error) { +func (n *layer) containerInfo(ctx context.Context, cid *cid.ID) (*cache.BucketInfo, error) { var ( err error res *container.Container rid = api.GetRequestID(ctx) bearerOpt = n.BearerOpt(ctx) - info = &BucketInfo{ + info = &cache.BucketInfo{ CID: cid, Name: cid.String(), } @@ -82,10 +74,17 @@ func (n *layer) containerInfo(ctx context.Context, cid *cid.ID) (*BucketInfo, er } } + if err := n.bucketCache.Put(info); err != nil { + n.log.Warn("could not put bucket info into cache", + zap.Stringer("cid", cid), + zap.String("bucket_name", info.Name), + zap.Error(err)) + } + return info, nil } -func (n *layer) containerList(ctx context.Context) ([]*BucketInfo, error) { +func (n *layer) containerList(ctx context.Context) ([]*cache.BucketInfo, error) { var ( err error own = n.Owner(ctx) @@ -101,7 +100,7 @@ func (n *layer) containerList(ctx context.Context) ([]*BucketInfo, error) { return nil, err } - list := make([]*BucketInfo, 0, len(res)) + list := make([]*cache.BucketInfo, 0, len(res)) for _, cid := range res { info, err := n.containerInfo(ctx, cid) if err != nil { diff --git a/api/layer/detector.go b/api/layer/detector.go index 3a9b7d85..81ec75b7 100644 --- a/api/layer/detector.go +++ b/api/layer/detector.go @@ -3,24 +3,56 @@ package layer import ( "io" "net/http" - "sync" ) -type detector struct { - io.Reader - sync.Once +type ( + detector struct { + io.Reader + err error + data []byte + } + errReader struct { + data []byte + err error + offset int + } +) - contentType string +const contentTypeDetectSize = 512 + +func newReader(data []byte, err error) *errReader { + return &errReader{data: data, err: err} } -func newDetector(r io.Reader) *detector { - return &detector{Reader: r} +func (r *errReader) Read(b []byte) (int, error) { + if r.offset >= len(r.data) { + return 0, io.EOF + } + n := copy(b, r.data[r.offset:]) + r.offset += n + if r.offset >= len(r.data) { + return n, r.err + } + return n, nil } -func (d *detector) Read(data []byte) (int, error) { - d.Do(func() { - d.contentType = http.DetectContentType(data) - }) - - return d.Reader.Read(data) +func newDetector(reader io.Reader) *detector { + return &detector{ + data: make([]byte, contentTypeDetectSize), + Reader: reader, + } +} + +func (d *detector) Detect() (string, error) { + n, err := d.Reader.Read(d.data) + if err != nil && err != io.EOF { + d.err = err + return "", err + } + d.data = d.data[:n] + return http.DetectContentType(d.data), nil +} + +func (d *detector) MultiReader() io.Reader { + return io.MultiReader(newReader(d.data, d.err), d.Reader) } diff --git a/api/layer/layer.go b/api/layer/layer.go index c454ac26..051629ea 100644 --- a/api/layer/layer.go +++ b/api/layer/layer.go @@ -28,10 +28,13 @@ import ( type ( layer struct { - pool pool.Pool - log *zap.Logger - listObjCache ObjectsListCache - objCache cache.ObjectsCache + pool pool.Pool + log *zap.Logger + listsCache ObjectsListCache + objCache cache.ObjectsCache + headCache cache.HeadObjectsCache + bucketCache cache.BucketCache + systemCache cache.SystemCache } // CacheConfig contains params for caches. @@ -156,8 +159,8 @@ type ( PutBucketVersioning(ctx context.Context, p *PutVersioningParams) (*ObjectInfo, error) GetBucketVersioning(ctx context.Context, name string) (*BucketSettings, error) - ListBuckets(ctx context.Context) ([]*BucketInfo, error) - GetBucketInfo(ctx context.Context, name string) (*BucketInfo, error) + ListBuckets(ctx context.Context) ([]*cache.BucketInfo, error) + GetBucketInfo(ctx context.Context, name string) (*cache.BucketInfo, error) GetBucketACL(ctx context.Context, name string) (*BucketACL, error) PutBucketACL(ctx context.Context, p *PutBucketACLParams) error CreateBucket(ctx context.Context, p *CreateBucketParams) (*cid.ID, error) @@ -228,26 +231,22 @@ func (v *objectVersions) getLast() *ObjectInfo { return nil } -func (v *objectVersions) getFiltered() []*ObjectVersionInfo { +func (v *objectVersions) getFiltered() []*ObjectInfo { if len(v.objects) == 0 { return nil } v.sort() existedVersions := getExistedVersions(v) - res := make([]*ObjectVersionInfo, 0, len(v.objects)) + res := make([]*ObjectInfo, 0, len(v.objects)) for _, version := range v.objects { delMark := version.Headers[versionsDeleteMarkAttr] if contains(existedVersions, version.Version()) && (delMark == delMarkFullObject || delMark == "") { - res = append(res, &ObjectVersionInfo{Object: version}) + res = append(res, version) } } - if len(res) > 0 { - res[len(res)-1].IsLatest = true - } - return res } @@ -279,10 +278,14 @@ func (t *VersionedObject) String() string { // and establishes gRPC connection with node. func NewLayer(log *zap.Logger, conns pool.Pool, config *CacheConfig) Client { return &layer{ - pool: conns, - log: log, - listObjCache: newListObjectsCache(config.ListObjectsLifetime), - objCache: cache.New(config.Size, config.Lifetime), + pool: conns, + log: log, + listsCache: newListObjectsCache(config.ListObjectsLifetime), + objCache: cache.New(config.Size, config.Lifetime), + //todo reconsider cache params + headCache: cache.NewHeadObject(1000, time.Minute), + bucketCache: cache.NewBucketCache(150, time.Minute), + systemCache: cache.NewSystemCache(1000, 5*time.Minute), } } @@ -320,12 +323,16 @@ func (n *layer) Get(ctx context.Context, address *object.Address) (*object.Objec } // GetBucketInfo returns bucket info by name. -func (n *layer) GetBucketInfo(ctx context.Context, name string) (*BucketInfo, error) { +func (n *layer) GetBucketInfo(ctx context.Context, name string) (*cache.BucketInfo, error) { name, err := url.QueryUnescape(name) if err != nil { return nil, err } + if bktInfo := n.bucketCache.Get(name); bktInfo != nil { + return bktInfo, nil + } + containerID := new(cid.ID) if err := containerID.Parse(name); err != nil { list, err := n.containerList(ctx) @@ -374,7 +381,7 @@ func (n *layer) PutBucketACL(ctx context.Context, param *PutBucketACLParams) err // ListBuckets returns all user containers. Name of the bucket is a container // id. Timestamp is omitted since it is not saved in neofs container. -func (n *layer) ListBuckets(ctx context.Context) ([]*BucketInfo, error) { +func (n *layer) ListBuckets(ctx context.Context) ([]*cache.BucketInfo, error) { return n.containerList(ctx) } @@ -382,21 +389,12 @@ func (n *layer) ListBuckets(ctx context.Context) ([]*BucketInfo, error) { func (n *layer) GetObject(ctx context.Context, p *GetObjectParams) error { var err error - //if bkt, err = n.GetBucketInfo(ctx, p.Bucket); err != nil { - // return fmt.Errorf("couldn't find bucket: %s : %w", p.Bucket, err) - //} else if oid, err = n.objectFindID(ctx, &findParams{cid: bkt.CID, val: p.Object}); err != nil { - // return fmt.Errorf("search of the object failed: cid: %s, val: %s : %w", bkt.CID, p.Object, err) - //} - - addr := object.NewAddress() - addr.SetObjectID(p.ObjectInfo.ID()) - addr.SetContainerID(p.ObjectInfo.CID()) - params := &getParams{ - Writer: p.Writer, - address: addr, - offset: p.Offset, - length: p.Length, + Writer: p.Writer, + cid: p.ObjectInfo.CID(), + oid: p.ObjectInfo.ID(), + offset: p.Offset, + length: p.Length, } if p.Range != nil { @@ -411,7 +409,7 @@ func (n *layer) GetObject(ctx context.Context, p *GetObjectParams) error { } if err != nil { - n.objCache.Delete(addr) + n.objCache.Delete(p.ObjectInfo.Address()) return fmt.Errorf("couldn't get object, cid: %s : %w", p.ObjectInfo.CID(), err) } @@ -433,32 +431,26 @@ func (n *layer) GetObjectInfo(ctx context.Context, p *HeadObjectParams) (*Object return n.headVersion(ctx, bkt, p.VersionID) } -func (n *layer) getSettingsObjectInfo(ctx context.Context, bkt *BucketInfo) (*ObjectInfo, error) { +func (n *layer) getSettingsObjectInfo(ctx context.Context, bkt *cache.BucketInfo) (*ObjectInfo, error) { + if meta := n.systemCache.Get(bktVersionSettingsObject); meta != nil { + return objInfoFromMeta(bkt, meta), nil + } + oid, err := n.objectFindID(ctx, &findParams{cid: bkt.CID, attr: objectSystemAttributeName, val: bktVersionSettingsObject}) if err != nil { return nil, err } - addr := object.NewAddress() - addr.SetObjectID(oid) - addr.SetContainerID(bkt.CID) - - /* todo: now we get an address via request to NeoFS and try to find the object with the address in cache - but it will be resolved after implementation of local cache with nicenames and address of objects - for get/head requests */ - meta := n.objCache.Get(addr) - if meta == nil { - meta, err = n.objectHead(ctx, bkt.CID, oid) - if err != nil { - n.log.Error("could not fetch object head", zap.Error(err)) - return nil, err - } - if err = n.objCache.Put(addr, *meta); err != nil { - n.log.Error("couldn't cache an object", zap.Error(err)) - } + meta, err := n.objectHead(ctx, bkt.CID, oid) + if err != nil { + n.log.Error("could not fetch object head", zap.Error(err)) + return nil, err + } + if err = n.systemCache.Put(bktVersionSettingsObject, meta); err != nil { + n.log.Error("couldn't cache system object", zap.Error(err)) } - return objectInfoFromMeta(bkt, meta, "", ""), nil + return objInfoFromMeta(bkt, meta), nil } // PutObject into storage. @@ -496,7 +488,7 @@ func (n *layer) CopyObject(ctx context.Context, p *CopyObjectParams) (*ObjectInf } // DeleteObject removes all objects with passed nice name. -func (n *layer) deleteObject(ctx context.Context, bkt *BucketInfo, obj *VersionedObject) error { +func (n *layer) deleteObject(ctx context.Context, bkt *cache.BucketInfo, obj *VersionedObject) error { var ( err error ids []*object.ID @@ -543,7 +535,7 @@ func (n *layer) deleteObject(ctx context.Context, bkt *BucketInfo, obj *Versione return nil } -func (n *layer) checkVersionsExists(ctx context.Context, bkt *BucketInfo, obj *VersionedObject) (*object.ID, error) { +func (n *layer) checkVersionsExists(ctx context.Context, bkt *cache.BucketInfo, obj *VersionedObject) (*object.ID, error) { id := object.NewID() if err := id.Parse(obj.VersionID); err != nil { return nil, &errors.DeleteError{Err: errors.GetAPIError(errors.ErrInvalidVersion), Object: obj.String()} @@ -608,60 +600,70 @@ func (n *layer) DeleteBucket(ctx context.Context, p *DeleteBucketParams) error { } func (n *layer) ListObjectVersions(ctx context.Context, p *ListObjectVersionsParams) (*ListObjectVersionsInfo, error) { - res := ListObjectVersionsInfo{} - versions := make(map[string]*objectVersions) + var versions map[string]*objectVersions + res := &ListObjectVersionsInfo{} bkt, err := n.GetBucketInfo(ctx, p.Bucket) if err != nil { return nil, err } - ids, err := n.objectSearch(ctx, &findParams{cid: bkt.CID}) + + cacheKey, err := createKey(ctx, bkt.CID, listVersionsMethod, p.Prefix, p.Delimiter) if err != nil { return nil, err } - for _, id := range ids { - meta, err := n.objectHead(ctx, bkt.CID, id) + allObjects := n.listsCache.Get(cacheKey) + if allObjects == nil { + versions, err = n.getAllObjectsVersions(ctx, bkt, p.Prefix, p.Delimiter) if err != nil { - n.log.Warn("could not fetch object meta", zap.Error(err)) - continue + return nil, err } - if oi := objectInfoFromMeta(bkt, meta, p.Prefix, p.Delimiter); oi != nil { - if oi.Name <= p.KeyMarker { - continue - } - if isSystem(oi) { - continue - } - if objVersions, ok := versions[oi.Name]; ok { - objVersions.appendVersion(oi) - versions[oi.Name] = objVersions - } else { - objVersion := newObjectVersions(oi.Name) - objVersion.appendVersion(oi) - versions[oi.Name] = objVersion - } + sortedNames := make([]string, 0, len(versions)) + for k := range versions { + sortedNames = append(sortedNames, k) } + sort.Strings(sortedNames) + + allObjects = make([]*ObjectInfo, 0, p.MaxKeys) + for _, name := range sortedNames { + allObjects = append(allObjects, versions[name].getFiltered()...) + } + + // putting to cache a copy of allObjects because allObjects can be modified further + n.listsCache.Put(cacheKey, append([]*ObjectInfo(nil), allObjects...)) } - sortedNames := make([]string, 0, len(versions)) - for k := range versions { - sortedNames = append(sortedNames, k) - } - sort.Strings(sortedNames) - - objects := make([]*ObjectVersionInfo, 0, p.MaxKeys) - for _, name := range sortedNames { - objects = append(objects, versions[name].getFiltered()...) - if len(objects) > p.MaxKeys { - objects = objects[:p.MaxKeys] + for i, obj := range allObjects { + if obj.Name >= p.KeyMarker && obj.Version() >= p.VersionIDMarker { + allObjects = allObjects[i:] break } } + res.CommonPrefixes, allObjects = triageObjects(allObjects) + + if len(allObjects) > p.MaxKeys { + res.IsTruncated = true + res.NextKeyMarker = allObjects[p.MaxKeys].Name + res.NextVersionIDMarker = allObjects[p.MaxKeys].Version() + + allObjects = allObjects[:p.MaxKeys] + res.KeyMarker = allObjects[p.MaxKeys-1].Name + res.VersionIDMarker = allObjects[p.MaxKeys-1].Version() + } + + objects := make([]*ObjectVersionInfo, len(allObjects)) + for i, obj := range allObjects { + objects[i] = &ObjectVersionInfo{Object: obj} + if i == len(allObjects)-1 || allObjects[i+1].Name != obj.Name { + objects[i].IsLatest = true + } + } + res.Version, res.DeleteMarker = triageVersions(objects) - return &res, nil + return res, nil } func sortVersions(versions []*ObjectInfo) { @@ -773,7 +775,7 @@ func (n *layer) GetBucketVersioning(ctx context.Context, bucketName string) (*Bu return n.getBucketSettings(ctx, bktInfo) } -func (n *layer) getBucketSettings(ctx context.Context, bktInfo *BucketInfo) (*BucketSettings, error) { +func (n *layer) getBucketSettings(ctx context.Context, bktInfo *cache.BucketInfo) (*BucketSettings, error) { objInfo, err := n.getSettingsObjectInfo(ctx, bktInfo) if err != nil { return nil, err diff --git a/api/layer/object.go b/api/layer/object.go index b418af03..226dec23 100644 --- a/api/layer/object.go +++ b/api/layer/object.go @@ -14,6 +14,8 @@ import ( cid "github.com/nspcc-dev/neofs-api-go/pkg/container/id" "github.com/nspcc-dev/neofs-api-go/pkg/object" "github.com/nspcc-dev/neofs-api-go/pkg/owner" + "github.com/nspcc-dev/neofs-s3-gw/api" + "github.com/nspcc-dev/neofs-s3-gw/api/cache" apiErrors "github.com/nspcc-dev/neofs-s3-gw/api/errors" "go.uber.org/zap" ) @@ -29,9 +31,10 @@ type ( io.Writer *object.Range - offset int64 - length int64 - address *object.Address + offset int64 + length int64 + cid *cid.ID + oid *object.ID } // ListObjectsParamsCommon contains common parameters for ListObjectsV1 and ListObjectsV2. @@ -58,7 +61,7 @@ type ( } allObjectParams struct { - Bucket *BucketInfo + Bucket *cache.BucketInfo Delimiter string Prefix string } @@ -96,12 +99,16 @@ func (n *layer) objectFindID(ctx context.Context, p *findParams) (*object.ID, er return nil, errors.New("several objects with the same name found") } -// objectHead returns all object's headers. -func (n *layer) objectHead(ctx context.Context, cid *cid.ID, oid *object.ID) (*object.Object, error) { +func newAddress(cid *cid.ID, oid *object.ID) *object.Address { address := object.NewAddress() address.SetContainerID(cid) address.SetObjectID(oid) - ops := new(client.ObjectHeaderParams).WithAddress(address).WithAllFields() + return address +} + +// objectHead returns all object's headers. +func (n *layer) objectHead(ctx context.Context, cid *cid.ID, oid *object.ID) (*object.Object, error) { + ops := new(client.ObjectHeaderParams).WithAddress(newAddress(cid, oid)).WithAllFields() return n.pool.GetObjectHeader(ctx, ops, n.BearerOpt(ctx)) } @@ -109,19 +116,19 @@ func (n *layer) objectHead(ctx context.Context, cid *cid.ID, oid *object.ID) (*o func (n *layer) objectGet(ctx context.Context, p *getParams) (*object.Object, error) { // prepare length/offset writer w := newWriter(p.Writer, p.offset, p.length) - ops := new(client.GetObjectParams).WithAddress(p.address).WithPayloadWriter(w) + ops := new(client.GetObjectParams).WithAddress(newAddress(p.cid, p.oid)).WithPayloadWriter(w) return n.pool.GetObject(ctx, ops, n.BearerOpt(ctx)) } // objectRange gets object range and writes it into provided io.Writer. func (n *layer) objectRange(ctx context.Context, p *getParams) ([]byte, error) { w := newWriter(p.Writer, p.offset, p.length) - ops := new(client.RangeDataParams).WithAddress(p.address).WithDataWriter(w).WithRange(p.Range) + ops := new(client.RangeDataParams).WithAddress(newAddress(p.cid, p.oid)).WithDataWriter(w).WithRange(p.Range) return n.pool.ObjectPayloadRangeData(ctx, ops, n.BearerOpt(ctx)) } // objectPut into NeoFS, took payload from io.Reader. -func (n *layer) objectPut(ctx context.Context, bkt *BucketInfo, p *PutObjectParams) (*ObjectInfo, error) { +func (n *layer) objectPut(ctx context.Context, bkt *cache.BucketInfo, p *PutObjectParams) (*ObjectInfo, error) { own := n.Owner(ctx) obj, err := url.QueryUnescape(p.Object) if err != nil { @@ -135,8 +142,15 @@ func (n *layer) objectPut(ctx context.Context, bkt *BucketInfo, p *PutObjectPara } idsToDeleteArr := updateCRDT2PSetHeaders(p, versions, versioningEnabled) + r := p.Reader + if len(p.Header[api.ContentType]) == 0 { + d := newDetector(p.Reader) + if contentType, err := d.Detect(); err == nil { + p.Header[api.ContentType] = contentType + } + r = d.MultiReader() + } rawObject := formRawObject(p, bkt.CID, own, obj) - r := newDetector(p.Reader) ops := new(client.PutObjectParams).WithObject(rawObject.Object()).WithPayloadReader(r) oid, err := n.pool.PutObject(ctx, ops, n.BearerOpt(ctx)) @@ -144,11 +158,21 @@ func (n *layer) objectPut(ctx context.Context, bkt *BucketInfo, p *PutObjectPara return nil, err } + if p.Header[versionsDeleteMarkAttr] == delMarkFullObject { + if last := versions.getLast(); last != nil { + n.objCache.Delete(last.Address()) + } + } + meta, err := n.objectHead(ctx, bkt.CID, oid) if err != nil { return nil, err } + if err = n.objCache.Put(*meta); err != nil { + n.log.Error("couldn't cache an object", zap.Error(err)) + } + for _, id := range idsToDeleteArr { if err = n.objectDelete(ctx, bkt.CID, id); err != nil { n.log.Warn("couldn't delete object", @@ -168,7 +192,7 @@ func (n *layer) objectPut(ctx context.Context, bkt *BucketInfo, p *PutObjectPara Created: time.Now(), CreationEpoch: meta.CreationEpoch(), Headers: p.Header, - ContentType: r.contentType, + ContentType: p.Header[api.ContentType], HashSum: meta.PayloadChecksum().String(), }, nil } @@ -229,9 +253,12 @@ func updateCRDT2PSetHeaders(p *PutObjectParams, versions *objectVersions, versio versionsDeletedStr += "," } - lastVersion := versions.getLast() - p.Header[versionsDelAttr] = versionsDeletedStr + lastVersion.Version() - idsToDeleteArr = append(idsToDeleteArr, lastVersion.ID()) + if lastVersion := versions.getLast(); lastVersion != nil { + p.Header[versionsDelAttr] = versionsDeletedStr + lastVersion.Version() + idsToDeleteArr = append(idsToDeleteArr, lastVersion.ID()) + } else if len(versionsDeletedStr) != 0 { + p.Header[versionsDelAttr] = versionsDeletedStr + } for _, version := range versions.objects { if contains(versions.delList, version.Version()) { @@ -243,7 +270,13 @@ func updateCRDT2PSetHeaders(p *PutObjectParams, versions *objectVersions, versio return idsToDeleteArr } -func (n *layer) headLastVersionIfNotDeleted(ctx context.Context, bkt *BucketInfo, objectName string) (*ObjectInfo, error) { +func (n *layer) headLastVersionIfNotDeleted(ctx context.Context, bkt *cache.BucketInfo, objectName string) (*ObjectInfo, error) { + if address := n.headCache.Get(bkt.Name + "/" + objectName); address != nil { + if headInfo := n.objCache.Get(address); headInfo != nil { + return objInfoFromMeta(bkt, headInfo), nil + } + } + versions, err := n.headVersions(ctx, bkt, objectName) if err != nil { return nil, err @@ -253,10 +286,17 @@ func (n *layer) headLastVersionIfNotDeleted(ctx context.Context, bkt *BucketInfo if lastVersion == nil { return nil, apiErrors.GetAPIError(apiErrors.ErrNoSuchKey) } + + if err = n.headCache.Put(lastVersion.NiceName(), lastVersion.Address()); err != nil { + n.log.Warn("couldn't put obj address to head cache", + zap.String("obj nice name", lastVersion.NiceName()), + zap.Error(err)) + } + return lastVersion, nil } -func (n *layer) headVersions(ctx context.Context, bkt *BucketInfo, objectName string) (*objectVersions, error) { +func (n *layer) headVersions(ctx context.Context, bkt *cache.BucketInfo, objectName string) (*objectVersions, error) { ids, err := n.objectSearch(ctx, &findParams{cid: bkt.CID, val: objectName}) if err != nil { return nil, err @@ -276,6 +316,13 @@ func (n *layer) headVersions(ctx context.Context, bkt *BucketInfo, objectName st zap.Error(err)) continue } + if err = n.objCache.Put(*meta); err != nil { + n.log.Warn("couldn't put meta to objects cache", + zap.Stringer("object id", id), + zap.Stringer("bucket id", bkt.CID), + zap.Error(err)) + } + if oi := objectInfoFromMeta(bkt, meta, "", ""); oi != nil { if isSystem(oi) { continue @@ -287,12 +334,16 @@ func (n *layer) headVersions(ctx context.Context, bkt *BucketInfo, objectName st return versions, nil } -func (n *layer) headVersion(ctx context.Context, bkt *BucketInfo, versionID string) (*ObjectInfo, error) { +func (n *layer) headVersion(ctx context.Context, bkt *cache.BucketInfo, versionID string) (*ObjectInfo, error) { oid := object.NewID() if err := oid.Parse(versionID); err != nil { return nil, err } + if headInfo := n.objCache.Get(newAddress(bkt.CID, oid)); headInfo != nil { + return objInfoFromMeta(bkt, headInfo), nil + } + meta, err := n.objectHead(ctx, bkt.CID, oid) if err != nil { if strings.Contains(err.Error(), "not found") { @@ -301,14 +352,22 @@ func (n *layer) headVersion(ctx context.Context, bkt *BucketInfo, versionID stri return nil, err } - return objectInfoFromMeta(bkt, meta, "", ""), nil + objInfo := objectInfoFromMeta(bkt, meta, "", "") + if err = n.objCache.Put(*meta); err != nil { + n.log.Warn("couldn't put obj to object cache", + zap.String("bucket name", objInfo.Bucket), + zap.Stringer("bucket cid", objInfo.CID()), + zap.String("object name", objInfo.Name), + zap.Stringer("object id", objInfo.ID()), + zap.Error(err)) + } + + return objInfo, nil } // objectDelete puts tombstone object into neofs. func (n *layer) objectDelete(ctx context.Context, cid *cid.ID, oid *object.ID) error { - address := object.NewAddress() - address.SetContainerID(cid) - address.SetObjectID(oid) + address := newAddress(cid, oid) dop := new(client.DeleteObjectParams) dop.WithAddress(address) n.objCache.Delete(address) @@ -390,35 +449,11 @@ func (n *layer) ListObjectsV2(ctx context.Context, p *ListObjectsParamsV2) (*Lis } func (n *layer) listSortedObjectsFromNeoFS(ctx context.Context, p allObjectParams) ([]*ObjectInfo, error) { - ids, err := n.objectSearch(ctx, &findParams{cid: p.Bucket.CID}) + versions, err := n.getAllObjectsVersions(ctx, p.Bucket, p.Prefix, p.Delimiter) if err != nil { return nil, err } - versions := make(map[string]*objectVersions, len(ids)/2) - - for _, id := range ids { - meta, err := n.objectHead(ctx, p.Bucket.CID, id) - if err != nil { - n.log.Warn("could not fetch object meta", zap.Error(err)) - continue - } - if oi := objectInfoFromMeta(p.Bucket, meta, p.Prefix, p.Delimiter); oi != nil { - if isSystem(oi) { - continue - } - - if objVersions, ok := versions[oi.Name]; ok { - objVersions.appendVersion(oi) - versions[oi.Name] = objVersions - } else { - objVersion := newObjectVersions(oi.Name) - objVersion.appendVersion(oi) - versions[oi.Name] = objVersion - } - } - } - objects := make([]*ObjectInfo, 0, len(versions)) for _, v := range versions { lastVersion := v.getLast() @@ -434,6 +469,38 @@ func (n *layer) listSortedObjectsFromNeoFS(ctx context.Context, p allObjectParam return objects, nil } +func (n *layer) getAllObjectsVersions(ctx context.Context, bkt *cache.BucketInfo, prefix, delimiter string) (map[string]*objectVersions, error) { + ids, err := n.objectSearch(ctx, &findParams{cid: bkt.CID}) + if err != nil { + return nil, err + } + + versions := make(map[string]*objectVersions, len(ids)/2) + for _, id := range ids { + meta, err := n.objectHead(ctx, bkt.CID, id) + if err != nil { + n.log.Warn("could not fetch object meta", zap.Error(err)) + continue + } + if oi := objectInfoFromMeta(bkt, meta, prefix, delimiter); oi != nil { + if isSystem(oi) { + continue + } + + if objVersions, ok := versions[oi.Name]; ok { + objVersions.appendVersion(oi) + versions[oi.Name] = objVersions + } else { + objVersion := newObjectVersions(oi.Name) + objVersion.appendVersion(oi) + versions[oi.Name] = objVersion + } + } + } + + return versions, nil +} + func getExistedVersions(versions *objectVersions) []string { var res []string for _, add := range versions.addList { @@ -498,7 +565,7 @@ func triageObjects(allObjects []*ObjectInfo) (prefixes []string, objects []*Obje func (n *layer) listAllObjects(ctx context.Context, p ListObjectsParamsCommon) ([]*ObjectInfo, error) { var ( err error - bkt *BucketInfo + bkt *cache.BucketInfo cacheKey cacheOptions allObjects []*ObjectInfo ) @@ -507,11 +574,11 @@ func (n *layer) listAllObjects(ctx context.Context, p ListObjectsParamsCommon) ( return nil, err } - if cacheKey, err = createKey(ctx, bkt.CID, p.Prefix, p.Delimiter); err != nil { + if cacheKey, err = createKey(ctx, bkt.CID, listObjectsMethod, p.Prefix, p.Delimiter); err != nil { return nil, err } - allObjects = n.listObjCache.Get(cacheKey) + allObjects = n.listsCache.Get(cacheKey) if allObjects == nil { allObjects, err = n.listSortedObjectsFromNeoFS(ctx, allObjectParams{ @@ -524,13 +591,13 @@ func (n *layer) listAllObjects(ctx context.Context, p ListObjectsParamsCommon) ( } // putting to cache a copy of allObjects because allObjects can be modified further - n.listObjCache.Put(cacheKey, append([]*ObjectInfo(nil), allObjects...)) + n.listsCache.Put(cacheKey, append([]*ObjectInfo(nil), allObjects...)) } return allObjects, nil } -func (n *layer) isVersioningEnabled(ctx context.Context, bktInfo *BucketInfo) bool { +func (n *layer) isVersioningEnabled(ctx context.Context, bktInfo *cache.BucketInfo) bool { settings, err := n.getBucketSettings(ctx, bktInfo) if err != nil { n.log.Warn("couldn't get versioning settings object", zap.Error(err)) diff --git a/api/layer/object_list_cache.go b/api/layer/object_list_cache.go index 6ed174a9..e67ac1ae 100644 --- a/api/layer/object_list_cache.go +++ b/api/layer/object_list_cache.go @@ -30,6 +30,11 @@ type ( // DefaultObjectsListCacheLifetime is a default lifetime of entries in cache of ListObjects. const DefaultObjectsListCacheLifetime = time.Second * 60 +const ( + listObjectsMethod = "listObjects" + listVersionsMethod = "listVersions" +) + type ( listObjectsCache struct { cacheLifetime time.Duration @@ -78,7 +83,7 @@ func (l *listObjectsCache) Put(key cacheOptions, objects []*ObjectInfo) { }) } -func createKey(ctx context.Context, cid *cid.ID, prefix, delimiter string) (cacheOptions, error) { +func createKey(ctx context.Context, cid *cid.ID, method, prefix, delimiter string) (cacheOptions, error) { box, err := GetBoxData(ctx) if err != nil { return cacheOptions{}, err diff --git a/api/layer/util.go b/api/layer/util.go index a915c5d3..557141f7 100644 --- a/api/layer/util.go +++ b/api/layer/util.go @@ -12,6 +12,7 @@ import ( "github.com/nspcc-dev/neofs-api-go/pkg/object" "github.com/nspcc-dev/neofs-api-go/pkg/owner" "github.com/nspcc-dev/neofs-s3-gw/api" + "github.com/nspcc-dev/neofs-s3-gw/api/cache" "github.com/nspcc-dev/neofs-s3-gw/creds/accessbox" ) @@ -60,7 +61,7 @@ type ( // ListObjectVersionsInfo stores info and list of objects' versions. ListObjectVersionsInfo struct { - CommonPrefixes []*string + CommonPrefixes []string IsTruncated bool KeyMarker string NextKeyMarker string @@ -84,7 +85,11 @@ func userHeaders(attrs []*object.Attribute) map[string]string { return result } -func objectInfoFromMeta(bkt *BucketInfo, meta *object.Object, prefix, delimiter string) *ObjectInfo { +func objInfoFromMeta(bkt *cache.BucketInfo, meta *object.Object) *ObjectInfo { + return objectInfoFromMeta(bkt, meta, "", "") +} + +func objectInfoFromMeta(bkt *cache.BucketInfo, meta *object.Object, prefix, delimiter string) *ObjectInfo { var ( isDir bool size int64 @@ -164,6 +169,12 @@ func (o *ObjectInfo) ID() *object.ID { return o.id } // Version returns object version from ObjectInfo. func (o *ObjectInfo) Version() string { return o.id.String() } +// NiceName returns object name for cache. +func (o *ObjectInfo) NiceName() string { return o.Bucket + "/" + o.Name } + +// Address returns object address. +func (o *ObjectInfo) Address() *object.Address { return newAddress(o.bucketID, o.id) } + // CID returns bucket ID from ObjectInfo. func (o *ObjectInfo) CID() *cid.ID { return o.bucketID } diff --git a/api/layer/util_test.go b/api/layer/util_test.go index d393b143..950b783c 100644 --- a/api/layer/util_test.go +++ b/api/layer/util_test.go @@ -9,6 +9,7 @@ import ( cid "github.com/nspcc-dev/neofs-api-go/pkg/container/id" "github.com/nspcc-dev/neofs-api-go/pkg/object" "github.com/nspcc-dev/neofs-api-go/pkg/owner" + "github.com/nspcc-dev/neofs-s3-gw/api/cache" "github.com/stretchr/testify/require" ) @@ -19,7 +20,7 @@ var ( defaultTestContentType = http.DetectContentType(defaultTestPayload) ) -func newTestObject(oid *object.ID, bkt *BucketInfo, name string) *object.Object { +func newTestObject(oid *object.ID, bkt *cache.BucketInfo, name string) *object.Object { filename := object.NewAttribute() filename.SetKey(object.AttributeFileName) filename.SetValue(name) @@ -43,7 +44,7 @@ func newTestObject(oid *object.ID, bkt *BucketInfo, name string) *object.Object return raw.Object() } -func newTestInfo(oid *object.ID, bkt *BucketInfo, name string, isDir bool) *ObjectInfo { +func newTestInfo(oid *object.ID, bkt *cache.BucketInfo, name string, isDir bool) *ObjectInfo { info := &ObjectInfo{ id: oid, Name: name, @@ -71,7 +72,7 @@ func Test_objectInfoFromMeta(t *testing.T) { oid := object.NewID() containerID := cid.New() - bkt := &BucketInfo{ + bkt := &cache.BucketInfo{ Name: "test-container", CID: containerID, Owner: uid, diff --git a/api/layer/versioning_test.go b/api/layer/versioning_test.go index 31ec3e04..e607ef5e 100644 --- a/api/layer/versioning_test.go +++ b/api/layer/versioning_test.go @@ -20,6 +20,7 @@ import ( "github.com/nspcc-dev/neofs-api-go/pkg/session" "github.com/nspcc-dev/neofs-api-go/pkg/token" "github.com/nspcc-dev/neofs-s3-gw/api" + "github.com/nspcc-dev/neofs-s3-gw/api/cache" "github.com/nspcc-dev/neofs-s3-gw/creds/accessbox" "github.com/nspcc-dev/neofs-sdk-go/pkg/logger" "github.com/nspcc-dev/neofs-sdk-go/pkg/pool" @@ -61,10 +62,7 @@ func (t *testPool) PutObject(ctx context.Context, params *client.PutObjectParams raw.SetPayload(all) } - addr := object.NewAddress() - addr.SetObjectID(raw.ID()) - addr.SetContainerID(raw.ContainerID()) - + addr := newAddress(raw.ContainerID(), raw.ID()) t.objects[addr.String()] = raw.Object() return raw.ID(), nil } @@ -329,8 +327,12 @@ func prepareContext(t *testing.T) *testContext { require.NoError(t, err) return &testContext{ - ctx: ctx, - layer: NewLayer(l, tp), + ctx: ctx, + layer: NewLayer(l, tp, &CacheConfig{ + Size: cache.DefaultObjectsCacheSize, + Lifetime: cache.DefaultObjectsCacheLifetime, + ListObjectsLifetime: DefaultObjectsListCacheLifetime}, + ), bkt: bktName, bktID: bktID, obj: "obj1", From 4bb885d5260aa67fbe10f18140c242efec1282fe Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Thu, 19 Aug 2021 09:55:22 +0300 Subject: [PATCH 10/10] [#122] Refactor Signed-off-by: Denis Kirillov --- api/cache/{bucket.go => buckets.go} | 21 +- api/cache/{head_cache.go => names.go} | 22 +- api/cache/object_cache_test.go | 1 - api/cache/system.go | 19 +- api/errors/errors.go | 28 +- api/handler/acl.go | 11 +- api/handler/copy.go | 40 ++- api/handler/delete.go | 32 ++- api/handler/get.go | 26 +- api/handler/head.go | 22 +- api/handler/object_list.go | 10 + api/handler/put.go | 6 + api/handler/response.go | 2 +- api/handler/versioning.go | 31 ++- api/headers.go | 5 + api/layer/container.go | 33 ++- api/layer/layer.go | 358 ++------------------------ api/layer/object.go | 18 +- api/layer/object_list_cache.go | 2 + api/layer/versioning.go | 325 +++++++++++++++++++++++ docs/aws_s3_compat.md | 4 +- 21 files changed, 550 insertions(+), 466 deletions(-) rename api/cache/{bucket.go => buckets.go} (74%) rename api/cache/{head_cache.go => names.go} (51%) create mode 100644 api/layer/versioning.go diff --git a/api/cache/bucket.go b/api/cache/buckets.go similarity index 74% rename from api/cache/bucket.go rename to api/cache/buckets.go index c6896bdb..667e3653 100644 --- a/api/cache/bucket.go +++ b/api/cache/buckets.go @@ -18,10 +18,11 @@ type ( // BucketInfo stores basic bucket data. BucketInfo struct { - Name string - CID *cid.ID - Owner *owner.ID - Created time.Time + Name string + CID *cid.ID + Owner *owner.ID + Created time.Time + BasicACL uint32 } // GetBucketCache contains cache with objects and lifetime of cache entries. @@ -62,3 +63,15 @@ func (o *GetBucketCache) Put(bkt *BucketInfo) error { func (o *GetBucketCache) Delete(key string) bool { return o.cache.Remove(key) } + +const bktVersionSettingsObject = ".s3-versioning-settings" + +// SettingsObjectName is system name for bucket settings file. +func (b *BucketInfo) SettingsObjectName() string { + return bktVersionSettingsObject +} + +// SettingsObjectKey is key to use in SystemCache. +func (b *BucketInfo) SettingsObjectKey() string { + return b.Name + bktVersionSettingsObject +} diff --git a/api/cache/head_cache.go b/api/cache/names.go similarity index 51% rename from api/cache/head_cache.go rename to api/cache/names.go index 08ac3d13..afb4b577 100644 --- a/api/cache/head_cache.go +++ b/api/cache/names.go @@ -7,30 +7,32 @@ import ( "github.com/nspcc-dev/neofs-api-go/pkg/object" ) -// HeadObjectsCache provides interface for lru cache for objects. -type HeadObjectsCache interface { +// ObjectsNameCache provides interface for lru cache for objects. +// This cache contains mapping nice name to object addresses. +// Key is bucketName+objectName. +type ObjectsNameCache interface { Get(key string) *object.Address Put(key string, address *object.Address) error Delete(key string) bool } type ( - // HeadObjectCache contains cache with objects and lifetime of cache entries. - HeadObjectCache struct { + // NameCache contains cache with objects and lifetime of cache entries. + NameCache struct { cache gcache.Cache lifetime time.Duration } ) -// NewHeadObject creates an object of ObjectHeadersCache. -func NewHeadObject(cacheSize int, lifetime time.Duration) *HeadObjectCache { +// NewObjectsNameCache creates an object of ObjectsNameCache. +func NewObjectsNameCache(cacheSize int, lifetime time.Duration) *NameCache { gc := gcache.New(cacheSize).LRU().Build() - return &HeadObjectCache{cache: gc, lifetime: lifetime} + return &NameCache{cache: gc, lifetime: lifetime} } // Get returns cached object. -func (o *HeadObjectCache) Get(key string) *object.Address { +func (o *NameCache) Get(key string) *object.Address { entry, err := o.cache.Get(key) if err != nil { return nil @@ -45,11 +47,11 @@ func (o *HeadObjectCache) Get(key string) *object.Address { } // Put puts an object to cache. -func (o *HeadObjectCache) Put(key string, address *object.Address) error { +func (o *NameCache) Put(key string, address *object.Address) error { return o.cache.SetWithExpire(key, address, o.lifetime) } // Delete deletes an object from cache. -func (o *HeadObjectCache) Delete(key string) bool { +func (o *NameCache) Delete(key string) bool { return o.cache.Remove(key) } diff --git a/api/cache/object_cache_test.go b/api/cache/object_cache_test.go index e307004a..40dfd87e 100644 --- a/api/cache/object_cache_test.go +++ b/api/cache/object_cache_test.go @@ -5,7 +5,6 @@ import ( "time" "github.com/nspcc-dev/neofs-api-go/pkg/object" - objecttest "github.com/nspcc-dev/neofs-api-go/pkg/object/test" "github.com/stretchr/testify/require" ) diff --git a/api/cache/system.go b/api/cache/system.go index d8f18b00..7ceec010 100644 --- a/api/cache/system.go +++ b/api/cache/system.go @@ -3,35 +3,36 @@ package cache import ( "time" - "github.com/nspcc-dev/neofs-api-go/pkg/object" - "github.com/bluele/gcache" + "github.com/nspcc-dev/neofs-api-go/pkg/object" ) type ( // SystemCache provides interface for lru cache for objects. + // This cache contains "system" objects (bucket versioning settings, tagging object etc.). + // Key is bucketName+systemFileName. SystemCache interface { Get(key string) *object.Object Put(key string, obj *object.Object) error Delete(key string) bool } - // systemCache contains cache with objects and lifetime of cache entries. - systemCache struct { + // SysCache contains cache with objects and lifetime of cache entries. + SysCache struct { cache gcache.Cache lifetime time.Duration } ) // NewSystemCache creates an object of SystemCache. -func NewSystemCache(cacheSize int, lifetime time.Duration) SystemCache { +func NewSystemCache(cacheSize int, lifetime time.Duration) *SysCache { gc := gcache.New(cacheSize).LRU().Build() - return &systemCache{cache: gc, lifetime: lifetime} + return &SysCache{cache: gc, lifetime: lifetime} } // Get returns cached object. -func (o *systemCache) Get(key string) *object.Object { +func (o *SysCache) Get(key string) *object.Object { entry, err := o.cache.Get(key) if err != nil { return nil @@ -46,11 +47,11 @@ func (o *systemCache) Get(key string) *object.Object { } // Put puts an object to cache. -func (o *systemCache) Put(key string, obj *object.Object) error { +func (o *SysCache) Put(key string, obj *object.Object) error { return o.cache.SetWithExpire(key, obj, o.lifetime) } // Delete deletes an object from cache. -func (o *systemCache) Delete(key string) bool { +func (o *SysCache) Delete(key string) bool { return o.cache.Remove(key) } diff --git a/api/errors/errors.go b/api/errors/errors.go index d1dd1126..f9c8aadb 100644 --- a/api/errors/errors.go +++ b/api/errors/errors.go @@ -1924,26 +1924,18 @@ func GetAPIError(code ErrorCode) Error { return errorCodes.toAPIErr(ErrInternalError) } -// GenericError - generic object layer error. -type GenericError struct { - Bucket string - Object string +// ObjectError - error that linked to specific object. +type ObjectError struct { + Err error + Object string + Version string } -// ObjectAlreadyExists object already exists. -// This type should be removed when s3-gw will support versioning. -type ObjectAlreadyExists GenericError - -func (e ObjectAlreadyExists) Error() string { - return "Object: " + e.Bucket + "#" + e.Object + " already exists" +func (e ObjectError) Error() string { + return fmt.Sprintf("%s (%s:%s)", e.Err, e.Object, e.Version) } -// DeleteError - returns when cant remove object. -type DeleteError struct { - Err error - Object string -} - -func (e DeleteError) Error() string { - return fmt.Sprintf("%s (%s)", e.Err, e.Object) +// ObjectVersion get "object:version" string. +func (e ObjectError) ObjectVersion() string { + return e.Object + ":" + e.Version } diff --git a/api/handler/acl.go b/api/handler/acl.go index 2131e0ca..b5b244b8 100644 --- a/api/handler/acl.go +++ b/api/handler/acl.go @@ -13,6 +13,7 @@ import ( "github.com/nspcc-dev/neofs-api-go/pkg/acl/eacl" "github.com/nspcc-dev/neofs-api-go/pkg/object" "github.com/nspcc-dev/neofs-s3-gw/api" + "github.com/nspcc-dev/neofs-s3-gw/api/cache" "github.com/nspcc-dev/neofs-s3-gw/api/errors" "github.com/nspcc-dev/neofs-s3-gw/api/layer" ) @@ -239,7 +240,13 @@ func (h *handler) PutObjectACLHandler(w http.ResponseWriter, r *http.Request) { return } - if _, err = h.obj.GetObjectInfo(r.Context(), reqInfo.BucketName, reqInfo.ObjectName); err != nil { + p := &layer.HeadObjectParams{ + Bucket: reqInfo.BucketName, + Object: reqInfo.ObjectName, + VersionID: reqInfo.URL.Query().Get(api.QueryVersionID), + } + + if _, err = h.obj.GetObjectInfo(r.Context(), p); err != nil { h.logAndSendError(w, "could not get object info", reqInfo, err) return } @@ -273,7 +280,7 @@ func (h *handler) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Request) } } -func checkOwner(info *layer.BucketInfo, owner string) error { +func checkOwner(info *cache.BucketInfo, owner string) error { if owner == "" { return nil } diff --git a/api/handler/copy.go b/api/handler/copy.go index 0503a837..86864fe1 100644 --- a/api/handler/copy.go +++ b/api/handler/copy.go @@ -32,10 +32,11 @@ func path2BucketObject(path string) (bucket, prefix string) { func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) { var ( err error - inf *layer.ObjectInfo + info *layer.ObjectInfo metadata map[string]string - reqInfo = api.GetReqInfo(r.Context()) + reqInfo = api.GetReqInfo(r.Context()) + versionID string ) src := r.Header.Get("X-Amz-Copy-Source") @@ -45,14 +46,7 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) { // of the version ID to null. If you have enabled versioning, Amazon S3 assigns a // unique version ID value for the object. if u, err := url.Parse(src); err == nil { - // Check if versionId query param was added, if yes then check if - // its non "null" value, we should error out since we do not support - // any versions other than "null". - if vid := u.Query().Get("versionId"); vid != "" && vid != "null" { - h.logAndSendError(w, "no such version", reqInfo, errors.GetAPIError(errors.ErrNoSuchVersion)) - return - } - + versionID = u.Query().Get(api.QueryVersionID) src = u.Path } @@ -66,7 +60,7 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) { p := &layer.HeadObjectParams{ Bucket: srcBucket, Object: srcObject, - VersionID: reqInfo.URL.Query().Get("versionId"), + VersionID: versionID, } if args.MetadataDirective == replaceMetadataDirective { @@ -85,46 +79,46 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) { return } - if inf, err = h.obj.GetObjectInfo(r.Context(), p); err != nil { + if info, err = h.obj.GetObjectInfo(r.Context(), p); err != nil { h.logAndSendError(w, "could not find object", reqInfo, err) return } - if err = checkPreconditions(inf, args.Conditional); err != nil { + if err = checkPreconditions(info, args.Conditional); err != nil { h.logAndSendError(w, "precondition failed", reqInfo, errors.GetAPIError(errors.ErrPreconditionFailed)) return } if metadata == nil { - if len(inf.ContentType) > 0 { - inf.Headers[api.ContentType] = inf.ContentType + if len(info.ContentType) > 0 { + info.Headers[api.ContentType] = info.ContentType } - metadata = inf.Headers + metadata = info.Headers } else if contentType := r.Header.Get(api.ContentType); len(contentType) > 0 { metadata[api.ContentType] = contentType } params := &layer.CopyObjectParams{ - SrcObject: inf, + SrcObject: info, DstBucket: reqInfo.BucketName, DstObject: reqInfo.ObjectName, - SrcSize: inf.Size, + SrcSize: info.Size, Header: metadata, } additional := []zap.Field{zap.String("src_bucket_name", srcBucket), zap.String("src_object_name", srcObject)} - if inf, err = h.obj.CopyObject(r.Context(), params); err != nil { + if info, err = h.obj.CopyObject(r.Context(), params); err != nil { h.logAndSendError(w, "couldn't copy object", reqInfo, err, additional...) return - } else if err = api.EncodeToResponse(w, &CopyObjectResponse{LastModified: inf.Created.Format(time.RFC3339), ETag: inf.HashSum}); err != nil { + } else if err = api.EncodeToResponse(w, &CopyObjectResponse{LastModified: info.Created.Format(time.RFC3339), ETag: info.HashSum}); err != nil { h.logAndSendError(w, "something went wrong", reqInfo, err, additional...) return } h.log.Info("object is copied", - zap.String("bucket", inf.Bucket), - zap.String("object", inf.Name), - zap.Stringer("object_id", inf.ID())) + zap.String("bucket", info.Bucket), + zap.String("object", info.Name), + zap.Stringer("object_id", info.ID())) } func parseCopyObjectArgs(headers http.Header) (*copyObjectArgs, error) { diff --git a/api/handler/delete.go b/api/handler/delete.go index a8b56bc6..15a4b225 100644 --- a/api/handler/delete.go +++ b/api/handler/delete.go @@ -27,9 +27,10 @@ type ObjectIdentifier struct { // DeleteError structure. type DeleteError struct { - Code string - Message string - Key string + Code string + Message string + Key string + VersionID string `xml:"versionId,omitempty"` } // DeleteObjectsResponse container for multiple object deletes. @@ -47,7 +48,7 @@ func (h *handler) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) { reqInfo := api.GetReqInfo(r.Context()) versionedObject := []*layer.VersionedObject{{ Name: reqInfo.ObjectName, - VersionID: reqInfo.URL.Query().Get("versionId"), + VersionID: reqInfo.URL.Query().Get(api.QueryVersionID), }} if err := h.checkBucketOwner(r, reqInfo.BucketName); err != nil { @@ -99,7 +100,7 @@ func (h *handler) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Re return } - removed := make(map[string]struct{}) + removed := make(map[string]*layer.VersionedObject) toRemove := make([]*layer.VersionedObject, 0, len(requested.Objects)) for _, obj := range requested.Objects { versionedObj := &layer.VersionedObject{ @@ -107,7 +108,7 @@ func (h *handler) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Re VersionID: obj.VersionID, } toRemove = append(toRemove, versionedObj) - removed[versionedObj.String()] = struct{}{} + removed[versionedObj.String()] = versionedObj } response := &DeleteObjectsResponse{ @@ -135,23 +136,26 @@ func (h *handler) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Re h.logAndSendError(w, "could not delete objects", reqInfo, nil, additional...) for _, e := range errs { - if err, ok := e.(*errors.DeleteError); ok { + if err, ok := e.(*errors.ObjectError); ok { code := "BadRequest" - desc := err.Error() + if s3err, ok := err.Err.(errors.Error); ok { + code = s3err.Code + } response.Errors = append(response.Errors, DeleteError{ - Code: code, - Message: desc, - Key: err.Object, + Code: code, + Message: err.Error(), + Key: err.Object, + VersionID: err.Version, }) - delete(removed, err.Object) + delete(removed, err.ObjectVersion()) } } } - for key := range removed { - response.DeletedObjects = append(response.DeletedObjects, ObjectIdentifier{ObjectName: key}) + for _, val := range removed { + response.DeletedObjects = append(response.DeletedObjects, ObjectIdentifier{ObjectName: val.Name, VersionID: val.VersionID}) } if err := api.EncodeToResponse(w, response); err != nil { diff --git a/api/handler/get.go b/api/handler/get.go index 9a1c62f0..7572423e 100644 --- a/api/handler/get.go +++ b/api/handler/get.go @@ -82,7 +82,7 @@ func writeHeaders(h http.Header, info *layer.ObjectInfo) { func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) { var ( err error - inf *layer.ObjectInfo + info *layer.ObjectInfo params *layer.RangeParams reqInfo = api.GetReqInfo(r.Context()) @@ -102,30 +102,30 @@ func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) { p := &layer.HeadObjectParams{ Bucket: reqInfo.BucketName, Object: reqInfo.ObjectName, - VersionID: reqInfo.URL.Query().Get("versionId"), + VersionID: reqInfo.URL.Query().Get(api.QueryVersionID), } - if inf, err = h.obj.GetObjectInfo(r.Context(), p); err != nil { + if info, err = h.obj.GetObjectInfo(r.Context(), p); err != nil { h.logAndSendError(w, "could not find object", reqInfo, err) return } - if err = checkPreconditions(inf, args.Conditional); err != nil { + if err = checkPreconditions(info, args.Conditional); err != nil { h.logAndSendError(w, "precondition failed", reqInfo, err) return } - if params, err = fetchRangeHeader(r.Header, uint64(inf.Size)); err != nil { + if params, err = fetchRangeHeader(r.Header, uint64(info.Size)); err != nil { h.logAndSendError(w, "could not parse range header", reqInfo, err) return } - writeHeaders(w.Header(), inf) + writeHeaders(w.Header(), info) if params != nil { - writeRangeHeaders(w, params, inf.Size) + writeRangeHeaders(w, params, info.Size) } getParams := &layer.GetObjectParams{ - ObjectInfo: inf, + ObjectInfo: info, Writer: w, Range: params, VersionID: p.VersionID, @@ -135,17 +135,17 @@ func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) { } } -func checkPreconditions(inf *layer.ObjectInfo, args *conditionalArgs) error { - if len(args.IfMatch) > 0 && args.IfMatch != inf.HashSum { +func checkPreconditions(info *layer.ObjectInfo, args *conditionalArgs) error { + if len(args.IfMatch) > 0 && args.IfMatch != info.HashSum { return errors.GetAPIError(errors.ErrPreconditionFailed) } - if len(args.IfNoneMatch) > 0 && args.IfNoneMatch == inf.HashSum { + if len(args.IfNoneMatch) > 0 && args.IfNoneMatch == info.HashSum { return errors.GetAPIError(errors.ErrNotModified) } - if args.IfModifiedSince != nil && inf.Created.Before(*args.IfModifiedSince) { + if args.IfModifiedSince != nil && info.Created.Before(*args.IfModifiedSince) { return errors.GetAPIError(errors.ErrNotModified) } - if args.IfUnmodifiedSince != nil && inf.Created.After(*args.IfUnmodifiedSince) { + if args.IfUnmodifiedSince != nil && info.Created.After(*args.IfUnmodifiedSince) { if len(args.IfMatch) == 0 { return errors.GetAPIError(errors.ErrPreconditionFailed) } diff --git a/api/handler/head.go b/api/handler/head.go index 1f0a3031..a03c33ca 100644 --- a/api/handler/head.go +++ b/api/handler/head.go @@ -25,8 +25,8 @@ func getRangeToDetectContentType(maxSize int64) *layer.RangeParams { func (h *handler) HeadObjectHandler(w http.ResponseWriter, r *http.Request) { var ( - err error - inf *layer.ObjectInfo + err error + info *layer.ObjectInfo reqInfo = api.GetReqInfo(r.Context()) ) @@ -39,30 +39,30 @@ func (h *handler) HeadObjectHandler(w http.ResponseWriter, r *http.Request) { p := &layer.HeadObjectParams{ Bucket: reqInfo.BucketName, Object: reqInfo.ObjectName, - VersionID: reqInfo.URL.Query().Get("versionId"), + VersionID: reqInfo.URL.Query().Get(api.QueryVersionID), } - if inf, err = h.obj.GetObjectInfo(r.Context(), p); err != nil { + if info, err = h.obj.GetObjectInfo(r.Context(), p); err != nil { h.logAndSendError(w, "could not fetch object info", reqInfo, err) return } - if len(inf.ContentType) == 0 { + if len(info.ContentType) == 0 { buffer := bytes.NewBuffer(make([]byte, 0, sizeToDetectType)) getParams := &layer.GetObjectParams{ - ObjectInfo: inf, + ObjectInfo: info, Writer: buffer, - Range: getRangeToDetectContentType(inf.Size), - VersionID: reqInfo.URL.Query().Get("versionId"), + Range: getRangeToDetectContentType(info.Size), + VersionID: reqInfo.URL.Query().Get(api.QueryVersionID), } if err = h.obj.GetObject(r.Context(), getParams); err != nil { - h.logAndSendError(w, "could not get object", reqInfo, err, zap.Stringer("oid", inf.ID())) + h.logAndSendError(w, "could not get object", reqInfo, err, zap.Stringer("oid", info.ID())) return } - inf.ContentType = http.DetectContentType(buffer.Bytes()) + info.ContentType = http.DetectContentType(buffer.Bytes()) } - writeHeaders(w.Header(), inf) + writeHeaders(w.Header(), info) w.WriteHeader(http.StatusOK) } diff --git a/api/handler/object_list.go b/api/handler/object_list.go index 293e6bb2..bafc62db 100644 --- a/api/handler/object_list.go +++ b/api/handler/object_list.go @@ -217,6 +217,16 @@ func (h *handler) ListBucketObjectVersionsHandler(w http.ResponseWriter, r *http return } + bktInfo, err := h.obj.GetBucketInfo(r.Context(), reqInfo.BucketName) + if err != nil { + h.logAndSendError(w, "could not get bucket info", reqInfo, err) + return + } + if err = checkOwner(bktInfo, r.Header.Get(api.AmzExpectedBucketOwner)); err != nil { + h.logAndSendError(w, "expected owner doesn't match", reqInfo, err) + return + } + info, err := h.obj.ListObjectVersions(r.Context(), p) if err != nil { h.logAndSendError(w, "something went wrong", reqInfo, err) diff --git a/api/handler/put.go b/api/handler/put.go index e4a89a51..288db285 100644 --- a/api/handler/put.go +++ b/api/handler/put.go @@ -116,6 +116,12 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) { } } + if versioning, err := h.obj.GetBucketVersioning(r.Context(), reqInfo.BucketName); err != nil { + h.log.Warn("couldn't get bucket versioning", zap.String("bucket name", reqInfo.BucketName), zap.Error(err)) + } else if versioning.VersioningEnabled { + w.Header().Set(api.AmzVersionID, info.Version()) + } + w.Header().Set(api.ETag, info.HashSum) api.WriteSuccessResponseHeadersOnly(w) } diff --git a/api/handler/response.go b/api/handler/response.go index 7d344891..a8884063 100644 --- a/api/handler/response.go +++ b/api/handler/response.go @@ -167,7 +167,7 @@ type ListObjectsVersionsResponse struct { // VersioningConfiguration contains VersioningConfiguration XML representation. type VersioningConfiguration struct { XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ VersioningConfiguration"` - Status string `xml:"Status"` + Status string `xml:"Status,omitempty"` MfaDelete string `xml:"MfaDelete,omitempty"` } diff --git a/api/handler/versioning.go b/api/handler/versioning.go index 19351a10..dbce63e9 100644 --- a/api/handler/versioning.go +++ b/api/handler/versioning.go @@ -24,6 +24,16 @@ func (h *handler) PutBucketVersioningHandler(w http.ResponseWriter, r *http.Requ Settings: &layer.BucketSettings{VersioningEnabled: configuration.Status == "Enabled"}, } + bktInfo, err := h.obj.GetBucketInfo(r.Context(), reqInfo.BucketName) + if err != nil { + h.logAndSendError(w, "could not get bucket info", reqInfo, err) + return + } + if err = checkOwner(bktInfo, r.Header.Get(api.AmzExpectedBucketOwner)); err != nil { + h.logAndSendError(w, "expected owner doesn't match", reqInfo, err) + return + } + if _, err := h.obj.PutBucketVersioning(r.Context(), p); err != nil { h.logAndSendError(w, "couldn't put update versioning settings", reqInfo, err) } @@ -33,14 +43,27 @@ func (h *handler) PutBucketVersioningHandler(w http.ResponseWriter, r *http.Requ func (h *handler) GetBucketVersioningHandler(w http.ResponseWriter, r *http.Request) { reqInfo := api.GetReqInfo(r.Context()) + bktInfo, err := h.obj.GetBucketInfo(r.Context(), reqInfo.BucketName) + if err != nil { + h.logAndSendError(w, "could not get bucket info", reqInfo, err) + return + } + if err = checkOwner(bktInfo, r.Header.Get(api.AmzExpectedBucketOwner)); err != nil { + h.logAndSendError(w, "expected owner doesn't match", reqInfo, err) + return + } + settings, err := h.obj.GetBucketVersioning(r.Context(), reqInfo.BucketName) if err != nil { + if errors.IsS3Error(err, errors.ErrNoSuchBucket) { + h.logAndSendError(w, "couldn't get versioning settings", reqInfo, err) + return + } h.log.Warn("couldn't get version settings object: default version settings will be used", zap.String("request_id", reqInfo.RequestID), zap.String("method", reqInfo.API), - zap.String("object_name", reqInfo.ObjectName), + zap.String("bucket_name", reqInfo.BucketName), zap.Error(err)) - return } if err = api.EncodeToResponse(w, formVersioningConfiguration(settings)); err != nil { @@ -49,12 +72,14 @@ func (h *handler) GetBucketVersioningHandler(w http.ResponseWriter, r *http.Requ } func formVersioningConfiguration(settings *layer.BucketSettings) *VersioningConfiguration { - res := &VersioningConfiguration{Status: "Suspended"} + res := &VersioningConfiguration{} if settings == nil { return res } if settings.VersioningEnabled { res.Status = "Enabled" + } else { + res.Status = "Suspended" } return res } diff --git a/api/headers.go b/api/headers.go index 767eca43..dfdefbdc 100644 --- a/api/headers.go +++ b/api/headers.go @@ -44,3 +44,8 @@ const ( ContainerID = "X-Container-Id" ) + +// S3 request query params. +const ( + QueryVersionID = "versionId" +) diff --git a/api/layer/container.go b/api/layer/container.go index 1e7b3b56..5f892217 100644 --- a/api/layer/container.go +++ b/api/layer/container.go @@ -117,29 +117,42 @@ func (n *layer) containerList(ctx context.Context) ([]*cache.BucketInfo, error) } func (n *layer) createContainer(ctx context.Context, p *CreateBucketParams) (*cid.ID, error) { + var err error + bktInfo := &cache.BucketInfo{ + Name: p.Name, + Owner: n.Owner(ctx), + Created: time.Now(), + BasicACL: p.ACL, + } cnr := container.New( container.WithPolicy(p.Policy), container.WithCustomBasicACL(p.ACL), container.WithAttribute(container.AttributeName, p.Name), - container.WithAttribute(container.AttributeTimestamp, strconv.FormatInt(time.Now().Unix(), 10))) + container.WithAttribute(container.AttributeTimestamp, strconv.FormatInt(bktInfo.Created.Unix(), 10))) cnr.SetSessionToken(p.BoxData.Gate.SessionToken) - cnr.SetOwnerID(n.Owner(ctx)) + cnr.SetOwnerID(bktInfo.Owner) - cid, err := n.pool.PutContainer(ctx, cnr) - if err != nil { - return nil, fmt.Errorf("failed to create a bucket: %w", err) - } - - if err = n.pool.WaitForContainerPresence(ctx, cid, pool.DefaultPollingParams()); err != nil { + if bktInfo.CID, err = n.pool.PutContainer(ctx, cnr); err != nil { return nil, err } - if err := n.setContainerEACLTable(ctx, cid, p.EACL); err != nil { + if err = n.pool.WaitForContainerPresence(ctx, bktInfo.CID, pool.DefaultPollingParams()); err != nil { return nil, err } - return cid, nil + if err = n.setContainerEACLTable(ctx, bktInfo.CID, p.EACL); err != nil { + return nil, err + } + + if err = n.bucketCache.Put(bktInfo); err != nil { + n.log.Warn("couldn't put bucket info into cache", + zap.String("bucket name", bktInfo.Name), + zap.Stringer("bucket cid", bktInfo.CID), + zap.Error(err)) + } + + return bktInfo.CID, nil } func (n *layer) setContainerEACLTable(ctx context.Context, cid *cid.ID, table *eacl.Table) error { diff --git a/api/layer/layer.go b/api/layer/layer.go index 051629ea..b49691bd 100644 --- a/api/layer/layer.go +++ b/api/layer/layer.go @@ -7,9 +7,6 @@ import ( "fmt" "io" "net/url" - "sort" - "strconv" - "strings" "time" "github.com/nspcc-dev/neofs-api-go/pkg/acl/eacl" @@ -32,7 +29,7 @@ type ( log *zap.Logger listsCache ObjectsListCache objCache cache.ObjectsCache - headCache cache.HeadObjectsCache + namesCache cache.ObjectsNameCache bucketCache cache.BucketCache systemCache cache.SystemCache } @@ -56,12 +53,10 @@ type ( GetObjectParams struct { Range *RangeParams ObjectInfo *ObjectInfo - //Bucket string - //Object string - Offset int64 - Length int64 - Writer io.Writer - VersionID string + Offset int64 + Length int64 + Writer io.Writer + VersionID string } // HeadObjectParams stores object head request parameters. @@ -139,14 +134,6 @@ type ( VersionID string } - objectVersions struct { - name string - objects []*ObjectInfo - addList []string - delList []string - isSorted bool - } - // NeoFS provides basic NeoFS interface. NeoFS interface { Get(ctx context.Context, address *object.Address) (*object.Object, error) @@ -181,95 +168,6 @@ type ( } ) -func newObjectVersions(name string) *objectVersions { - return &objectVersions{name: name} -} - -func (v *objectVersions) appendVersion(oi *ObjectInfo) { - addVers := append(splitVersions(oi.Headers[versionsAddAttr]), oi.Version()) - delVers := splitVersions(oi.Headers[versionsDelAttr]) - v.objects = append(v.objects, oi) - for _, add := range addVers { - if !contains(v.addList, add) { - v.addList = append(v.addList, add) - } - } - for _, del := range delVers { - if !contains(v.delList, del) { - v.delList = append(v.delList, del) - } - } - v.isSorted = false -} - -func (v *objectVersions) sort() { - if !v.isSorted { - sortVersions(v.objects) - v.isSorted = true - } -} - -func (v *objectVersions) getLast() *ObjectInfo { - if len(v.objects) == 0 { - return nil - } - - v.sort() - existedVersions := getExistedVersions(v) - for i := len(v.objects) - 1; i >= 0; i-- { - if contains(existedVersions, v.objects[i].Version()) { - delMarkHeader := v.objects[i].Headers[versionsDeleteMarkAttr] - if delMarkHeader == "" { - return v.objects[i] - } - if delMarkHeader == delMarkFullObject { - return nil - } - } - } - - return nil -} - -func (v *objectVersions) getFiltered() []*ObjectInfo { - if len(v.objects) == 0 { - return nil - } - - v.sort() - existedVersions := getExistedVersions(v) - res := make([]*ObjectInfo, 0, len(v.objects)) - - for _, version := range v.objects { - delMark := version.Headers[versionsDeleteMarkAttr] - if contains(existedVersions, version.Version()) && (delMark == delMarkFullObject || delMark == "") { - res = append(res, version) - } - } - - return res -} - -func (v *objectVersions) getAddHeader() string { - return strings.Join(v.addList, ",") -} - -func (v *objectVersions) getDelHeader() string { - return strings.Join(v.delList, ",") -} - -const ( - unversionedObjectVersionID = "null" - bktVersionSettingsObject = ".s3-versioning-settings" - objectSystemAttributeName = "S3-System-name" - attrVersionsIgnore = "S3-Versions-ignore" - attrSettingsVersioningEnabled = "S3-Settings-Versioning-enabled" - versionsDelAttr = "S3-Versions-del" - versionsAddAttr = "S3-Versions-add" - versionsDeleteMarkAttr = "S3-Versions-delete-mark" - delMarkFullObject = "*" -) - func (t *VersionedObject) String() string { return t.Name + ":" + t.VersionID } @@ -283,7 +181,7 @@ func NewLayer(log *zap.Logger, conns pool.Pool, config *CacheConfig) Client { listsCache: newListObjectsCache(config.ListObjectsLifetime), objCache: cache.New(config.Size, config.Lifetime), //todo reconsider cache params - headCache: cache.NewHeadObject(1000, time.Minute), + namesCache: cache.NewObjectsNameCache(1000, time.Minute), bucketCache: cache.NewBucketCache(150, time.Minute), systemCache: cache.NewSystemCache(1000, 5*time.Minute), } @@ -432,11 +330,11 @@ func (n *layer) GetObjectInfo(ctx context.Context, p *HeadObjectParams) (*Object } func (n *layer) getSettingsObjectInfo(ctx context.Context, bkt *cache.BucketInfo) (*ObjectInfo, error) { - if meta := n.systemCache.Get(bktVersionSettingsObject); meta != nil { + if meta := n.systemCache.Get(bkt.SettingsObjectKey()); meta != nil { return objInfoFromMeta(bkt, meta), nil } - oid, err := n.objectFindID(ctx, &findParams{cid: bkt.CID, attr: objectSystemAttributeName, val: bktVersionSettingsObject}) + oid, err := n.objectFindID(ctx, &findParams{cid: bkt.CID, attr: objectSystemAttributeName, val: bkt.SettingsObjectName()}) if err != nil { return nil, err } @@ -446,7 +344,7 @@ func (n *layer) getSettingsObjectInfo(ctx context.Context, bkt *cache.BucketInfo n.log.Error("could not fetch object head", zap.Error(err)) return nil, err } - if err = n.systemCache.Put(bktVersionSettingsObject, meta); err != nil { + if err = n.systemCache.Put(bkt.SettingsObjectKey(), meta); err != nil { n.log.Error("couldn't cache system object", zap.Error(err)) } @@ -506,7 +404,7 @@ func (n *layer) deleteObject(ctx context.Context, bkt *cache.BucketInfo, obj *Ve Header: map[string]string{versionsDeleteMarkAttr: obj.VersionID}, } if len(obj.VersionID) != 0 { - id, err := n.checkVersionsExists(ctx, bkt, obj) + id, err := n.checkVersionsExist(ctx, bkt, obj) if err != nil { return err } @@ -517,41 +415,24 @@ func (n *layer) deleteObject(ctx context.Context, bkt *cache.BucketInfo, obj *Ve p.Header[versionsDeleteMarkAttr] = delMarkFullObject } if _, err = n.objectPut(ctx, bkt, p); err != nil { - return &errors.DeleteError{Err: err, Object: obj.String()} + return err } } else { ids, err = n.objectSearch(ctx, &findParams{cid: bkt.CID, val: obj.Name}) if err != nil { - return &errors.DeleteError{Err: err, Object: obj.String()} + return err } } for _, id := range ids { if err = n.objectDelete(ctx, bkt.CID, id); err != nil { - return &errors.DeleteError{Err: err, Object: obj.String()} + return err } } return nil } -func (n *layer) checkVersionsExists(ctx context.Context, bkt *cache.BucketInfo, obj *VersionedObject) (*object.ID, error) { - id := object.NewID() - if err := id.Parse(obj.VersionID); err != nil { - return nil, &errors.DeleteError{Err: errors.GetAPIError(errors.ErrInvalidVersion), Object: obj.String()} - } - - versions, err := n.headVersions(ctx, bkt, obj.Name) - if err != nil { - return nil, &errors.DeleteError{Err: err, Object: obj.String()} - } - if !contains(getExistedVersions(versions), obj.VersionID) { - return nil, &errors.DeleteError{Err: errors.GetAPIError(errors.ErrInvalidVersion), Object: obj.String()} - } - - return id, nil -} - // DeleteObjects from the storage. func (n *layer) DeleteObjects(ctx context.Context, bucket string, objects []*VersionedObject) []error { var errs = make([]error, 0, len(objects)) @@ -561,9 +442,9 @@ func (n *layer) DeleteObjects(ctx context.Context, bucket string, objects []*Ver return append(errs, err) } - for i := range objects { - if err := n.deleteObject(ctx, bkt, objects[i]); err != nil { - errs = append(errs, err) + for _, obj := range objects { + if err := n.deleteObject(ctx, bkt, obj); err != nil { + errs = append(errs, &errors.ObjectError{Err: err, Object: obj.Name, Version: obj.VersionID}) } } @@ -588,210 +469,17 @@ func (n *layer) DeleteBucket(ctx context.Context, p *DeleteBucketParams) error { return err } - ids, err := n.objectSearch(ctx, &findParams{cid: bucketInfo.CID}) + objects, err := n.listSortedObjectsFromNeoFS(ctx, allObjectParams{Bucket: bucketInfo}) if err != nil { return err } - if len(ids) != 0 { + if len(objects) != 0 { return errors.GetAPIError(errors.ErrBucketNotEmpty) } - return n.deleteContainer(ctx, bucketInfo.CID) -} - -func (n *layer) ListObjectVersions(ctx context.Context, p *ListObjectVersionsParams) (*ListObjectVersionsInfo, error) { - var versions map[string]*objectVersions - res := &ListObjectVersionsInfo{} - - bkt, err := n.GetBucketInfo(ctx, p.Bucket) - if err != nil { - return nil, err - } - - cacheKey, err := createKey(ctx, bkt.CID, listVersionsMethod, p.Prefix, p.Delimiter) - if err != nil { - return nil, err - } - - allObjects := n.listsCache.Get(cacheKey) - if allObjects == nil { - versions, err = n.getAllObjectsVersions(ctx, bkt, p.Prefix, p.Delimiter) - if err != nil { - return nil, err - } - - sortedNames := make([]string, 0, len(versions)) - for k := range versions { - sortedNames = append(sortedNames, k) - } - sort.Strings(sortedNames) - - allObjects = make([]*ObjectInfo, 0, p.MaxKeys) - for _, name := range sortedNames { - allObjects = append(allObjects, versions[name].getFiltered()...) - } - - // putting to cache a copy of allObjects because allObjects can be modified further - n.listsCache.Put(cacheKey, append([]*ObjectInfo(nil), allObjects...)) - } - - for i, obj := range allObjects { - if obj.Name >= p.KeyMarker && obj.Version() >= p.VersionIDMarker { - allObjects = allObjects[i:] - break - } - } - - res.CommonPrefixes, allObjects = triageObjects(allObjects) - - if len(allObjects) > p.MaxKeys { - res.IsTruncated = true - res.NextKeyMarker = allObjects[p.MaxKeys].Name - res.NextVersionIDMarker = allObjects[p.MaxKeys].Version() - - allObjects = allObjects[:p.MaxKeys] - res.KeyMarker = allObjects[p.MaxKeys-1].Name - res.VersionIDMarker = allObjects[p.MaxKeys-1].Version() - } - - objects := make([]*ObjectVersionInfo, len(allObjects)) - for i, obj := range allObjects { - objects[i] = &ObjectVersionInfo{Object: obj} - if i == len(allObjects)-1 || allObjects[i+1].Name != obj.Name { - objects[i].IsLatest = true - } - } - - res.Version, res.DeleteMarker = triageVersions(objects) - return res, nil -} - -func sortVersions(versions []*ObjectInfo) { - sort.Slice(versions, func(i, j int) bool { - return less(versions[i], versions[j]) - }) -} - -func triageVersions(objVersions []*ObjectVersionInfo) ([]*ObjectVersionInfo, []*ObjectVersionInfo) { - if len(objVersions) == 0 { - return nil, nil - } - - var resVersion []*ObjectVersionInfo - var resDelMarkVersions []*ObjectVersionInfo - - for _, version := range objVersions { - if version.Object.Headers[versionsDeleteMarkAttr] == delMarkFullObject { - resDelMarkVersions = append(resDelMarkVersions, version) - } else { - resVersion = append(resVersion, version) - } - } - - return resVersion, resDelMarkVersions -} - -func less(ov1, ov2 *ObjectInfo) bool { - if ov1.CreationEpoch == ov2.CreationEpoch { - return ov1.Version() < ov2.Version() - } - return ov1.CreationEpoch < ov2.CreationEpoch -} - -func contains(list []string, elem string) bool { - for _, item := range list { - if elem == item { - return true - } - } - return false -} - -func (n *layer) PutBucketVersioning(ctx context.Context, p *PutVersioningParams) (*ObjectInfo, error) { - bucketInfo, err := n.GetBucketInfo(ctx, p.Bucket) - if err != nil { - return nil, err - } - - objectInfo, err := n.getSettingsObjectInfo(ctx, bucketInfo) - if err != nil { - n.log.Warn("couldn't get bucket version settings object, new one will be created", - zap.String("bucket_name", bucketInfo.Name), - zap.Stringer("cid", bucketInfo.CID), - zap.Error(err)) - } - - attributes := make([]*object.Attribute, 0, 3) - - filename := object.NewAttribute() - filename.SetKey(objectSystemAttributeName) - filename.SetValue(bktVersionSettingsObject) - - createdAt := object.NewAttribute() - createdAt.SetKey(object.AttributeTimestamp) - createdAt.SetValue(strconv.FormatInt(time.Now().UTC().Unix(), 10)) - - versioningIgnore := object.NewAttribute() - versioningIgnore.SetKey(attrVersionsIgnore) - versioningIgnore.SetValue(strconv.FormatBool(true)) - - settingsVersioningEnabled := object.NewAttribute() - settingsVersioningEnabled.SetKey(attrSettingsVersioningEnabled) - settingsVersioningEnabled.SetValue(strconv.FormatBool(p.Settings.VersioningEnabled)) - - attributes = append(attributes, filename, createdAt, versioningIgnore, settingsVersioningEnabled) - - raw := object.NewRaw() - raw.SetOwnerID(bucketInfo.Owner) - raw.SetContainerID(bucketInfo.CID) - raw.SetAttributes(attributes...) - - ops := new(client.PutObjectParams).WithObject(raw.Object()) - oid, err := n.pool.PutObject(ctx, ops, n.BearerOpt(ctx)) - if err != nil { - return nil, err - } - - meta, err := n.objectHead(ctx, bucketInfo.CID, oid) - if err != nil { - return nil, err - } - - if objectInfo != nil { - if err = n.objectDelete(ctx, bucketInfo.CID, objectInfo.ID()); err != nil { - return nil, err - } - } - - return objectInfoFromMeta(bucketInfo, meta, "", ""), nil -} - -func (n *layer) GetBucketVersioning(ctx context.Context, bucketName string) (*BucketSettings, error) { - bktInfo, err := n.GetBucketInfo(ctx, bucketName) - if err != nil { - return nil, err - } - - return n.getBucketSettings(ctx, bktInfo) -} - -func (n *layer) getBucketSettings(ctx context.Context, bktInfo *cache.BucketInfo) (*BucketSettings, error) { - objInfo, err := n.getSettingsObjectInfo(ctx, bktInfo) - if err != nil { - return nil, err - } - - return objectInfoToBucketSettings(objInfo), nil -} - -func objectInfoToBucketSettings(info *ObjectInfo) *BucketSettings { - res := &BucketSettings{} - - enabled, ok := info.Headers["S3-Settings-Versioning-enabled"] - if ok { - if parsed, err := strconv.ParseBool(enabled); err == nil { - res.VersioningEnabled = parsed - } - } - return res + if err = n.deleteContainer(ctx, bucketInfo.CID); err != nil { + return err + } + n.bucketCache.Delete(bucketInfo.Name) + return nil } diff --git a/api/layer/object.go b/api/layer/object.go index 226dec23..b18c12cd 100644 --- a/api/layer/object.go +++ b/api/layer/object.go @@ -144,7 +144,7 @@ func (n *layer) objectPut(ctx context.Context, bkt *cache.BucketInfo, p *PutObje r := p.Reader if len(p.Header[api.ContentType]) == 0 { - d := newDetector(p.Reader) + d := newDetector(r) if contentType, err := d.Detect(); err == nil { p.Header[api.ContentType] = contentType } @@ -271,7 +271,7 @@ func updateCRDT2PSetHeaders(p *PutObjectParams, versions *objectVersions, versio } func (n *layer) headLastVersionIfNotDeleted(ctx context.Context, bkt *cache.BucketInfo, objectName string) (*ObjectInfo, error) { - if address := n.headCache.Get(bkt.Name + "/" + objectName); address != nil { + if address := n.namesCache.Get(bkt.Name + "/" + objectName); address != nil { if headInfo := n.objCache.Get(address); headInfo != nil { return objInfoFromMeta(bkt, headInfo), nil } @@ -287,7 +287,7 @@ func (n *layer) headLastVersionIfNotDeleted(ctx context.Context, bkt *cache.Buck return nil, apiErrors.GetAPIError(apiErrors.ErrNoSuchKey) } - if err = n.headCache.Put(lastVersion.NiceName(), lastVersion.Address()); err != nil { + if err = n.namesCache.Put(lastVersion.NiceName(), lastVersion.Address()); err != nil { n.log.Warn("couldn't put obj address to head cache", zap.String("obj nice name", lastVersion.NiceName()), zap.Error(err)) @@ -487,14 +487,12 @@ func (n *layer) getAllObjectsVersions(ctx context.Context, bkt *cache.BucketInfo continue } - if objVersions, ok := versions[oi.Name]; ok { - objVersions.appendVersion(oi) - versions[oi.Name] = objVersions - } else { - objVersion := newObjectVersions(oi.Name) - objVersion.appendVersion(oi) - versions[oi.Name] = objVersion + objVersions, ok := versions[oi.Name] + if !ok { + objVersions = newObjectVersions(oi.Name) } + objVersions.appendVersion(oi) + versions[oi.Name] = objVersions } } diff --git a/api/layer/object_list_cache.go b/api/layer/object_list_cache.go index e67ac1ae..18e562ce 100644 --- a/api/layer/object_list_cache.go +++ b/api/layer/object_list_cache.go @@ -45,6 +45,7 @@ type ( list []*ObjectInfo } cacheOptions struct { + method string key string delimiter string prefix string @@ -89,6 +90,7 @@ func createKey(ctx context.Context, cid *cid.ID, method, prefix, delimiter strin return cacheOptions{}, err } p := cacheOptions{ + method: method, key: box.Gate.AccessKey + cid.String(), delimiter: delimiter, prefix: prefix, diff --git a/api/layer/versioning.go b/api/layer/versioning.go new file mode 100644 index 00000000..20e5219a --- /dev/null +++ b/api/layer/versioning.go @@ -0,0 +1,325 @@ +package layer + +import ( + "context" + "sort" + "strconv" + "strings" + "time" + + "github.com/nspcc-dev/neofs-api-go/pkg/client" + "github.com/nspcc-dev/neofs-api-go/pkg/object" + "github.com/nspcc-dev/neofs-s3-gw/api/cache" + "github.com/nspcc-dev/neofs-s3-gw/api/errors" + "go.uber.org/zap" +) + +type objectVersions struct { + name string + objects []*ObjectInfo + addList []string + delList []string + isSorted bool +} + +const ( + unversionedObjectVersionID = "null" + objectSystemAttributeName = "S3-System-name" + attrVersionsIgnore = "S3-Versions-ignore" + attrSettingsVersioningEnabled = "S3-Settings-Versioning-enabled" + versionsDelAttr = "S3-Versions-del" + versionsAddAttr = "S3-Versions-add" + versionsDeleteMarkAttr = "S3-Versions-delete-mark" + delMarkFullObject = "*" +) + +func newObjectVersions(name string) *objectVersions { + return &objectVersions{name: name} +} + +func (v *objectVersions) appendVersion(oi *ObjectInfo) { + addVers := append(splitVersions(oi.Headers[versionsAddAttr]), oi.Version()) + delVers := splitVersions(oi.Headers[versionsDelAttr]) + v.objects = append(v.objects, oi) + for _, add := range addVers { + if !contains(v.addList, add) { + v.addList = append(v.addList, add) + } + } + for _, del := range delVers { + if !contains(v.delList, del) { + v.delList = append(v.delList, del) + } + } + v.isSorted = false +} + +func (v *objectVersions) sort() { + if !v.isSorted { + sort.Slice(v.objects, func(i, j int) bool { + return less(v.objects[i], v.objects[j]) + }) + v.isSorted = true + } +} + +func (v *objectVersions) getLast() *ObjectInfo { + if len(v.objects) == 0 { + return nil + } + + v.sort() + existedVersions := getExistedVersions(v) + for i := len(v.objects) - 1; i >= 0; i-- { + if contains(existedVersions, v.objects[i].Version()) { + delMarkHeader := v.objects[i].Headers[versionsDeleteMarkAttr] + if delMarkHeader == "" { + return v.objects[i] + } + if delMarkHeader == delMarkFullObject { + return nil + } + } + } + + return nil +} + +func (v *objectVersions) getFiltered() []*ObjectInfo { + if len(v.objects) == 0 { + return nil + } + + v.sort() + existedVersions := getExistedVersions(v) + res := make([]*ObjectInfo, 0, len(v.objects)) + + for _, version := range v.objects { + delMark := version.Headers[versionsDeleteMarkAttr] + if contains(existedVersions, version.Version()) && (delMark == delMarkFullObject || delMark == "") { + res = append(res, version) + } + } + + return res +} + +func (v *objectVersions) getAddHeader() string { + return strings.Join(v.addList, ",") +} + +func (v *objectVersions) getDelHeader() string { + return strings.Join(v.delList, ",") +} + +func (n *layer) PutBucketVersioning(ctx context.Context, p *PutVersioningParams) (*ObjectInfo, error) { + bucketInfo, err := n.GetBucketInfo(ctx, p.Bucket) + if err != nil { + return nil, err + } + + objectInfo, err := n.getSettingsObjectInfo(ctx, bucketInfo) + if err != nil { + n.log.Warn("couldn't get bucket version settings object, new one will be created", + zap.String("bucket_name", bucketInfo.Name), + zap.Stringer("cid", bucketInfo.CID), + zap.Error(err)) + } + + attributes := make([]*object.Attribute, 0, 3) + + filename := object.NewAttribute() + filename.SetKey(objectSystemAttributeName) + filename.SetValue(bucketInfo.SettingsObjectName()) + + createdAt := object.NewAttribute() + createdAt.SetKey(object.AttributeTimestamp) + createdAt.SetValue(strconv.FormatInt(time.Now().UTC().Unix(), 10)) + + versioningIgnore := object.NewAttribute() + versioningIgnore.SetKey(attrVersionsIgnore) + versioningIgnore.SetValue(strconv.FormatBool(true)) + + settingsVersioningEnabled := object.NewAttribute() + settingsVersioningEnabled.SetKey(attrSettingsVersioningEnabled) + settingsVersioningEnabled.SetValue(strconv.FormatBool(p.Settings.VersioningEnabled)) + + attributes = append(attributes, filename, createdAt, versioningIgnore, settingsVersioningEnabled) + + raw := object.NewRaw() + raw.SetOwnerID(bucketInfo.Owner) + raw.SetContainerID(bucketInfo.CID) + raw.SetAttributes(attributes...) + + ops := new(client.PutObjectParams).WithObject(raw.Object()) + oid, err := n.pool.PutObject(ctx, ops, n.BearerOpt(ctx)) + if err != nil { + return nil, err + } + + meta, err := n.objectHead(ctx, bucketInfo.CID, oid) + if err != nil { + return nil, err + } + + if err = n.systemCache.Put(bucketInfo.SettingsObjectKey(), meta); err != nil { + n.log.Error("couldn't cache system object", zap.Error(err)) + } + + if objectInfo != nil { + if err = n.objectDelete(ctx, bucketInfo.CID, objectInfo.ID()); err != nil { + return nil, err + } + } + + return objectInfoFromMeta(bucketInfo, meta, "", ""), nil +} + +func (n *layer) GetBucketVersioning(ctx context.Context, bucketName string) (*BucketSettings, error) { + bktInfo, err := n.GetBucketInfo(ctx, bucketName) + if err != nil { + return nil, err + } + + return n.getBucketSettings(ctx, bktInfo) +} + +func (n *layer) ListObjectVersions(ctx context.Context, p *ListObjectVersionsParams) (*ListObjectVersionsInfo, error) { + var versions map[string]*objectVersions + res := &ListObjectVersionsInfo{} + + bkt, err := n.GetBucketInfo(ctx, p.Bucket) + if err != nil { + return nil, err + } + + cacheKey, err := createKey(ctx, bkt.CID, listVersionsMethod, p.Prefix, p.Delimiter) + if err != nil { + return nil, err + } + + allObjects := n.listsCache.Get(cacheKey) + if allObjects == nil { + versions, err = n.getAllObjectsVersions(ctx, bkt, p.Prefix, p.Delimiter) + if err != nil { + return nil, err + } + + sortedNames := make([]string, 0, len(versions)) + for k := range versions { + sortedNames = append(sortedNames, k) + } + sort.Strings(sortedNames) + + allObjects = make([]*ObjectInfo, 0, p.MaxKeys) + for _, name := range sortedNames { + allObjects = append(allObjects, versions[name].getFiltered()...) + } + + // putting to cache a copy of allObjects because allObjects can be modified further + n.listsCache.Put(cacheKey, append([]*ObjectInfo(nil), allObjects...)) + } + + for i, obj := range allObjects { + if obj.Name >= p.KeyMarker && obj.Version() >= p.VersionIDMarker { + allObjects = allObjects[i:] + break + } + } + + res.CommonPrefixes, allObjects = triageObjects(allObjects) + + if len(allObjects) > p.MaxKeys { + res.IsTruncated = true + res.NextKeyMarker = allObjects[p.MaxKeys].Name + res.NextVersionIDMarker = allObjects[p.MaxKeys].Version() + + allObjects = allObjects[:p.MaxKeys] + res.KeyMarker = allObjects[p.MaxKeys-1].Name + res.VersionIDMarker = allObjects[p.MaxKeys-1].Version() + } + + objects := make([]*ObjectVersionInfo, len(allObjects)) + for i, obj := range allObjects { + objects[i] = &ObjectVersionInfo{Object: obj} + if i == len(allObjects)-1 || allObjects[i+1].Name != obj.Name { + objects[i].IsLatest = true + } + } + + res.Version, res.DeleteMarker = triageVersions(objects) + return res, nil +} + +func triageVersions(objVersions []*ObjectVersionInfo) ([]*ObjectVersionInfo, []*ObjectVersionInfo) { + if len(objVersions) == 0 { + return nil, nil + } + + var resVersion []*ObjectVersionInfo + var resDelMarkVersions []*ObjectVersionInfo + + for _, version := range objVersions { + if version.Object.Headers[versionsDeleteMarkAttr] == delMarkFullObject { + resDelMarkVersions = append(resDelMarkVersions, version) + } else { + resVersion = append(resVersion, version) + } + } + + return resVersion, resDelMarkVersions +} + +func less(ov1, ov2 *ObjectInfo) bool { + if ov1.CreationEpoch == ov2.CreationEpoch { + return ov1.Version() < ov2.Version() + } + return ov1.CreationEpoch < ov2.CreationEpoch +} + +func contains(list []string, elem string) bool { + for _, item := range list { + if elem == item { + return true + } + } + return false +} + +func (n *layer) getBucketSettings(ctx context.Context, bktInfo *cache.BucketInfo) (*BucketSettings, error) { + objInfo, err := n.getSettingsObjectInfo(ctx, bktInfo) + if err != nil { + return nil, err + } + + return objectInfoToBucketSettings(objInfo), nil +} + +func objectInfoToBucketSettings(info *ObjectInfo) *BucketSettings { + res := &BucketSettings{} + + enabled, ok := info.Headers[attrSettingsVersioningEnabled] + if ok { + if parsed, err := strconv.ParseBool(enabled); err == nil { + res.VersioningEnabled = parsed + } + } + return res +} + +func (n *layer) checkVersionsExist(ctx context.Context, bkt *cache.BucketInfo, obj *VersionedObject) (*object.ID, error) { + id := object.NewID() + if err := id.Parse(obj.VersionID); err != nil { + return nil, errors.GetAPIError(errors.ErrInvalidVersion) + } + + versions, err := n.headVersions(ctx, bkt, obj.Name) + if err != nil { + return nil, err + } + if !contains(getExistedVersions(versions), obj.VersionID) { + return nil, errors.GetAPIError(errors.ErrInvalidVersion) + } + + return id, nil +} diff --git a/docs/aws_s3_compat.md b/docs/aws_s3_compat.md index bb1eb533..647f8ce4 100644 --- a/docs/aws_s3_compat.md +++ b/docs/aws_s3_compat.md @@ -226,8 +226,8 @@ See also `GetObject` and other method parameters. | | Method | Comments | |----|---------------------|----------| -| 🔴 | GetBucketVersioning | | -| 🔴 | PutBucketVersioning | | +| 🟢 | GetBucketVersioning | | +| 🟢 | PutBucketVersioning | | ## Website