diff --git a/api/data/info.go b/api/data/info.go index 8cda706..445f786 100644 --- a/api/data/info.go +++ b/api/data/info.go @@ -47,6 +47,14 @@ type ( Headers map[string]string } + // NotificationInfo store info to send s3 notification. + NotificationInfo struct { + Name string + Version string + Size int64 + HashSum string + } + // BucketSettings stores settings such as versioning. BucketSettings struct { Versioning string `json:"versioning"` @@ -70,6 +78,16 @@ type ( } ) +// NotificationInfoFromObject creates new NotificationInfo from ObjectInfo. +func NotificationInfoFromObject(objInfo *ObjectInfo) *NotificationInfo { + return &NotificationInfo{ + Name: objInfo.Name, + Version: objInfo.Version(), + Size: objInfo.Size, + HashSum: objInfo.HashSum, + } +} + // SettingsObjectName is a system name for a bucket settings file. func (b *BucketInfo) SettingsObjectName() string { return bktSettingsObject } diff --git a/api/data/tree.go b/api/data/tree.go index 2fb8b46..951b48f 100644 --- a/api/data/tree.go +++ b/api/data/tree.go @@ -36,6 +36,8 @@ type BaseNodeVersion struct { ID uint64 OID oid.ID Timestamp uint64 + Size int64 + ETag string FilePath string } diff --git a/api/handler/acl.go b/api/handler/acl.go index 4c24508..62381c4 100644 --- a/api/handler/acl.go +++ b/api/handler/acl.go @@ -376,10 +376,10 @@ func (h *handler) PutObjectACLHandler(w http.ResponseWriter, r *http.Request) { } if updated { s := &SendNotificationParams{ - Event: EventObjectACLPut, - ObjInfo: extendedInfo.ObjectInfo, - BktInfo: bktInfo, - ReqInfo: reqInfo, + Event: EventObjectACLPut, + NotificationInfo: data.NotificationInfoFromObject(extendedInfo.ObjectInfo), + BktInfo: bktInfo, + ReqInfo: reqInfo, } if err = h.sendNotifications(r.Context(), s); err != nil { h.log.Error("couldn't send notification: %w", zap.Error(err)) diff --git a/api/handler/copy.go b/api/handler/copy.go index d260430..36b2339 100644 --- a/api/handler/copy.go +++ b/api/handler/copy.go @@ -7,6 +7,7 @@ import ( "time" "github.com/nspcc-dev/neofs-s3-gw/api" + "github.com/nspcc-dev/neofs-s3-gw/api/data" "github.com/nspcc-dev/neofs-s3-gw/api/errors" "github.com/nspcc-dev/neofs-s3-gw/api/layer" "github.com/nspcc-dev/neofs-sdk-go/session" @@ -165,10 +166,10 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) { zap.Stringer("object_id", info.ID)) s := &SendNotificationParams{ - Event: EventObjectCreatedCopy, - ObjInfo: info, - BktInfo: dstBktInfo, - ReqInfo: reqInfo, + Event: EventObjectCreatedCopy, + NotificationInfo: data.NotificationInfoFromObject(info), + BktInfo: dstBktInfo, + ReqInfo: reqInfo, } if err = h.sendNotifications(r.Context(), s); err != nil { h.log.Error("couldn't send notification: %w", zap.Error(err)) diff --git a/api/handler/delete.go b/api/handler/delete.go index f1bcec5..9ddbcc4 100644 --- a/api/handler/delete.go +++ b/api/handler/delete.go @@ -101,7 +101,7 @@ func (h *handler) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) { if bktSettings.VersioningEnabled() && len(versionID) == 0 { m = &SendNotificationParams{ Event: EventObjectRemovedDeleteMarkerCreated, - ObjInfo: &data.ObjectInfo{ + NotificationInfo: &data.NotificationInfo{ Name: reqInfo.ObjectName, HashSum: deletedObject.DeleteMarkerEtag, }, @@ -118,9 +118,9 @@ func (h *handler) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) { m = &SendNotificationParams{ Event: EventObjectRemovedDelete, - ObjInfo: &data.ObjectInfo{ - Name: reqInfo.ObjectName, - ID: objID, + NotificationInfo: &data.NotificationInfo{ + Name: reqInfo.ObjectName, + Version: objID.EncodeToString(), }, BktInfo: bktInfo, ReqInfo: reqInfo, diff --git a/api/handler/multipart_upload.go b/api/handler/multipart_upload.go index 6963a9c..4cf090c 100644 --- a/api/handler/multipart_upload.go +++ b/api/handler/multipart_upload.go @@ -9,6 +9,7 @@ import ( "github.com/google/uuid" "github.com/nspcc-dev/neofs-s3-gw/api" + "github.com/nspcc-dev/neofs-s3-gw/api/data" "github.com/nspcc-dev/neofs-s3-gw/api/errors" "github.com/nspcc-dev/neofs-s3-gw/api/layer" "github.com/nspcc-dev/neofs-sdk-go/session" @@ -364,7 +365,7 @@ func (h *handler) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http. ObjectName: objInfo.Name, VersionID: objInfo.Version(), } - if err = h.obj.PutObjectTagging(r.Context(), t, uploadData.TagSet); err != nil { + if _, err = h.obj.PutObjectTagging(r.Context(), t, uploadData.TagSet); err != nil { h.logAndSendError(w, "could not put tagging file of completed multipart upload", reqInfo, err, additional...) return } @@ -398,10 +399,10 @@ func (h *handler) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http. } s := &SendNotificationParams{ - Event: EventObjectCreatedCompleteMultipartUpload, - ObjInfo: objInfo, - BktInfo: bktInfo, - ReqInfo: reqInfo, + Event: EventObjectCreatedCompleteMultipartUpload, + NotificationInfo: data.NotificationInfoFromObject(objInfo), + BktInfo: bktInfo, + ReqInfo: reqInfo, } if err = h.sendNotifications(r.Context(), s); err != nil { h.log.Error("couldn't send notification: %w", zap.Error(err)) diff --git a/api/handler/notifications.go b/api/handler/notifications.go index a208ed7..29529ad 100644 --- a/api/handler/notifications.go +++ b/api/handler/notifications.go @@ -17,11 +17,11 @@ import ( type ( SendNotificationParams struct { - Event string - ObjInfo *data.ObjectInfo - BktInfo *data.BucketInfo - ReqInfo *api.ReqInfo - User string + Event string + NotificationInfo *data.NotificationInfo + BktInfo *data.BucketInfo + ReqInfo *api.ReqInfo + User string } NotificationConfiguration struct { @@ -163,7 +163,7 @@ func (h *handler) sendNotifications(ctx context.Context, p *SendNotificationPara p.User = bearer.ResolveIssuer(*box.Gate.BearerToken).EncodeToString() } - topics := filterSubjects(conf, p.Event, p.ObjInfo.Name) + topics := filterSubjects(conf, p.Event, p.NotificationInfo.Name) return h.notificator.SendNotifications(topics, p) } diff --git a/api/handler/put.go b/api/handler/put.go index ea360a8..d65e7d5 100644 --- a/api/handler/put.go +++ b/api/handler/put.go @@ -236,10 +236,10 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) { } s := &SendNotificationParams{ - Event: EventObjectCreatedPut, - ObjInfo: info, - BktInfo: bktInfo, - ReqInfo: reqInfo, + Event: EventObjectCreatedPut, + NotificationInfo: data.NotificationInfoFromObject(info), + BktInfo: bktInfo, + ReqInfo: reqInfo, } if err = h.sendNotifications(r.Context(), s); err != nil { h.log.Error("couldn't send notification: %w", zap.Error(err)) @@ -258,7 +258,7 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) { VersionID: info.Version(), } if tagSet != nil { - if err = h.obj.PutObjectTagging(r.Context(), t, tagSet); err != nil { + if _, err = h.obj.PutObjectTagging(r.Context(), t, tagSet); err != nil { h.logAndSendError(w, "could not upload object tagging", reqInfo, err) return } @@ -358,10 +358,10 @@ func (h *handler) PostObject(w http.ResponseWriter, r *http.Request) { } s := &SendNotificationParams{ - Event: EventObjectCreatedPost, - ObjInfo: info, - BktInfo: bktInfo, - ReqInfo: reqInfo, + Event: EventObjectCreatedPost, + NotificationInfo: data.NotificationInfoFromObject(info), + BktInfo: bktInfo, + ReqInfo: reqInfo, } if err = h.sendNotifications(r.Context(), s); err != nil { h.log.Error("couldn't send notification: %w", zap.Error(err)) @@ -386,7 +386,7 @@ func (h *handler) PostObject(w http.ResponseWriter, r *http.Request) { } if tagSet != nil { - if err = h.obj.PutObjectTagging(r.Context(), t, tagSet); err != nil { + if _, err = h.obj.PutObjectTagging(r.Context(), t, tagSet); err != nil { h.logAndSendError(w, "could not upload object tagging", reqInfo, err) return } diff --git a/api/handler/tagging.go b/api/handler/tagging.go index 23e2724..0436a82 100644 --- a/api/handler/tagging.go +++ b/api/handler/tagging.go @@ -41,18 +41,22 @@ func (h *handler) PutObjectTaggingHandler(w http.ResponseWriter, r *http.Request p := &layer.ObjectVersion{ BktInfo: bktInfo, ObjectName: reqInfo.ObjectName, - VersionID: reqInfo.URL.Query().Get("versionId"), + VersionID: reqInfo.URL.Query().Get(api.QueryVersionID), } - if err = h.obj.PutObjectTagging(r.Context(), p, tagSet); err != nil { + nodeVersion, err := h.obj.PutObjectTagging(r.Context(), p, tagSet) + if err != nil { h.logAndSendError(w, "could not put object tagging", reqInfo, err) return } s := &SendNotificationParams{ Event: EventObjectTaggingPut, - ObjInfo: &data.ObjectInfo{ - Name: reqInfo.ObjectName, + NotificationInfo: &data.NotificationInfo{ + Name: nodeVersion.FilePath, + Size: nodeVersion.Size, + Version: nodeVersion.OID.EncodeToString(), + HashSum: nodeVersion.ETag, }, BktInfo: bktInfo, ReqInfo: reqInfo, @@ -111,18 +115,22 @@ func (h *handler) DeleteObjectTaggingHandler(w http.ResponseWriter, r *http.Requ p := &layer.ObjectVersion{ BktInfo: bktInfo, ObjectName: reqInfo.ObjectName, - VersionID: reqInfo.URL.Query().Get("versionId"), + VersionID: reqInfo.URL.Query().Get(api.QueryVersionID), } - if err = h.obj.DeleteObjectTagging(r.Context(), p); err != nil { + nodeVersion, err := h.obj.DeleteObjectTagging(r.Context(), p) + if err != nil { h.logAndSendError(w, "could not delete object tagging", reqInfo, err) return } s := &SendNotificationParams{ Event: EventObjectTaggingDelete, - ObjInfo: &data.ObjectInfo{ - Name: reqInfo.ObjectName, + NotificationInfo: &data.NotificationInfo{ + Name: nodeVersion.FilePath, + Size: nodeVersion.Size, + Version: nodeVersion.OID.EncodeToString(), + HashSum: nodeVersion.ETag, }, BktInfo: bktInfo, ReqInfo: reqInfo, diff --git a/api/layer/layer.go b/api/layer/layer.go index e61c789..93d50ab 100644 --- a/api/layer/layer.go +++ b/api/layer/layer.go @@ -226,8 +226,8 @@ type ( DeleteBucketTagging(ctx context.Context, cnrID cid.ID) error GetObjectTagging(ctx context.Context, p *ObjectVersion) (string, map[string]string, error) - PutObjectTagging(ctx context.Context, p *ObjectVersion, tagSet map[string]string) error - DeleteObjectTagging(ctx context.Context, p *ObjectVersion) error + PutObjectTagging(ctx context.Context, p *ObjectVersion, tagSet map[string]string) (*data.NodeVersion, error) + DeleteObjectTagging(ctx context.Context, p *ObjectVersion) (*data.NodeVersion, error) PutObject(ctx context.Context, p *PutObjectParams) (*data.ObjectInfo, error) diff --git a/api/layer/object.go b/api/layer/object.go index a420cc8..9d51d9b 100644 --- a/api/layer/object.go +++ b/api/layer/object.go @@ -155,8 +155,11 @@ func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Object } newVersion := &data.NodeVersion{ - BaseNodeVersion: data.BaseNodeVersion{FilePath: p.Object}, - IsUnversioned: !bktSettings.VersioningEnabled(), + BaseNodeVersion: data.BaseNodeVersion{ + FilePath: p.Object, + Size: p.Size, + }, + IsUnversioned: !bktSettings.VersioningEnabled(), } r := p.Reader @@ -194,6 +197,7 @@ func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Object } newVersion.OID = id + newVersion.ETag = hex.EncodeToString(hash) if err = n.treeService.AddVersion(ctx, p.BktInfo.CID, newVersion); err != nil { return nil, fmt.Errorf("couldn't add new verion to tree service: %w", err) } @@ -223,7 +227,7 @@ func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Object Created: time.Now(), Headers: p.Header, ContentType: p.Header[api.ContentType], - HashSum: hex.EncodeToString(hash), + HashSum: newVersion.ETag, } if err = n.objCache.PutObject(objInfo); err != nil { diff --git a/api/layer/tagging.go b/api/layer/tagging.go index bde9a95..a86670e 100644 --- a/api/layer/tagging.go +++ b/api/layer/tagging.go @@ -42,45 +42,45 @@ func (n *layer) GetObjectTagging(ctx context.Context, p *ObjectVersion) (string, return p.VersionID, tags, nil } -func (n *layer) PutObjectTagging(ctx context.Context, p *ObjectVersion, tagSet map[string]string) error { +func (n *layer) PutObjectTagging(ctx context.Context, p *ObjectVersion, tagSet map[string]string) (*data.NodeVersion, error) { version, err := n.getNodeVersion(ctx, p) if err != nil { - return err + return nil, err } p.VersionID = version.OID.EncodeToString() err = n.treeService.PutObjectTagging(ctx, p.BktInfo.CID, version, tagSet) if err != nil { if errorsStd.Is(err, ErrNodeNotFound) { - return errors.GetAPIError(errors.ErrNoSuchKey) + return nil, errors.GetAPIError(errors.ErrNoSuchKey) } - return err + return nil, err } if err = n.systemCache.PutTagging(objectTaggingCacheKey(p), tagSet); err != nil { n.log.Error("couldn't cache system object", zap.Error(err)) } - return nil + return version, nil } -func (n *layer) DeleteObjectTagging(ctx context.Context, p *ObjectVersion) error { +func (n *layer) DeleteObjectTagging(ctx context.Context, p *ObjectVersion) (*data.NodeVersion, error) { version, err := n.getNodeVersion(ctx, p) if err != nil { - return err + return nil, err } err = n.treeService.DeleteObjectTagging(ctx, p.BktInfo.CID, version) if err != nil { if errorsStd.Is(err, ErrNodeNotFound) { - return errors.GetAPIError(errors.ErrNoSuchKey) + return nil, errors.GetAPIError(errors.ErrNoSuchKey) } - return err + return nil, err } n.systemCache.Delete(objectTaggingCacheKey(p)) - return nil + return version, nil } func (n *layer) GetBucketTagging(ctx context.Context, cnrID cid.ID) (map[string]string, error) { diff --git a/api/notifications/controller.go b/api/notifications/controller.go index 8e7531c..ad85694 100644 --- a/api/notifications/controller.go +++ b/api/notifications/controller.go @@ -240,10 +240,10 @@ func prepareEvent(p *handler.SendNotificationParams) *Event { Arn: p.BktInfo.Name, }, Object: Object{ - Key: p.ObjInfo.Name, - Size: p.ObjInfo.Size, - VersionID: p.ObjInfo.Version(), - ETag: p.ObjInfo.HashSum, + Key: p.NotificationInfo.Name, + Size: p.NotificationInfo.Size, + VersionID: p.NotificationInfo.Version, + ETag: p.NotificationInfo.HashSum, Sequencer: "", }, }, diff --git a/internal/neofs/tree.go b/internal/neofs/tree.go index 012b658..fe6d521 100644 --- a/internal/neofs/tree.go +++ b/internal/neofs/tree.go @@ -33,6 +33,7 @@ type ( ID uint64 ObjID oid.ID TimeStamp uint64 + Size int64 Meta map[string]string } @@ -112,26 +113,31 @@ type NodeResponse interface { } func newTreeNode(nodeInfo NodeResponse) (*TreeNode, error) { - var objID oid.ID - meta := make(map[string]string, len(nodeInfo.GetMeta())) - - for _, kv := range nodeInfo.GetMeta() { - if kv.GetKey() == oidKV { - if err := objID.DecodeString(string(kv.GetValue())); err != nil { - return nil, err - } - continue - } - - meta[kv.GetKey()] = string(kv.GetValue()) + treeNode := &TreeNode{ + ID: nodeInfo.GetNodeId(), + TimeStamp: nodeInfo.GetTimestamp(), + Meta: make(map[string]string, len(nodeInfo.GetMeta())), } - return &TreeNode{ - ID: nodeInfo.GetNodeId(), - ObjID: objID, - TimeStamp: nodeInfo.GetTimestamp(), - Meta: meta, - }, nil + for _, kv := range nodeInfo.GetMeta() { + switch kv.GetKey() { + case oidKV: + if err := treeNode.ObjID.DecodeString(string(kv.GetValue())); err != nil { + return nil, err + } + case sizeKV: + if sizeStr := string(kv.GetValue()); len(sizeStr) > 0 { + var err error + if treeNode.Size, err = strconv.ParseInt(sizeStr, 10, 64); err != nil { + return nil, fmt.Errorf("invalid size value '%s': %w", sizeStr, err) + } + } + default: + treeNode.Meta[kv.GetKey()] = string(kv.GetValue()) + } + } + + return treeNode, nil } func (n *TreeNode) Get(key string) (string, bool) { @@ -160,12 +166,15 @@ func newNodeVersion(filePath string, node NodeResponse) (*data.NodeVersion, erro func newNodeVersionFromTreeNode(filePath string, treeNode *TreeNode) *data.NodeVersion { _, isUnversioned := treeNode.Get(isUnversionedKV) _, isDeleteMarker := treeNode.Get(isDeleteMarkerKV) + eTag, _ := treeNode.Get(etagKV) version := &data.NodeVersion{ BaseNodeVersion: data.BaseNodeVersion{ ID: treeNode.ID, OID: treeNode.ObjID, Timestamp: treeNode.TimeStamp, + ETag: eTag, + Size: treeNode.Size, FilePath: filePath, }, IsUnversioned: isUnversioned, @@ -528,7 +537,7 @@ func (c *TreeClient) GetVersions(ctx context.Context, cnrID cid.ID, filepath str } func (c *TreeClient) GetLatestVersion(ctx context.Context, cnrID cid.ID, objectName string) (*data.NodeVersion, error) { - meta := []string{oidKV, isUnversionedKV, isDeleteMarkerKV} + meta := []string{oidKV, isUnversionedKV, isDeleteMarkerKV, etagKV, sizeKV} path := pathFromName(objectName) p := &getNodesParams{ @@ -1034,6 +1043,13 @@ func (c *TreeClient) addVersion(ctx context.Context, cnrID cid.ID, treeID string fileNameKV: path[len(path)-1], } + if version.Size > 0 { + meta[sizeKV] = strconv.FormatInt(version.Size, 10) + } + if len(version.ETag) > 0 { + meta[etagKV] = version.ETag + } + if version.DeleteMarker != nil { meta[isDeleteMarkerKV] = "true" meta[ownerKV] = version.DeleteMarker.Owner.EncodeToString() @@ -1062,7 +1078,7 @@ func (c *TreeClient) addVersion(ctx context.Context, cnrID cid.ID, treeID string } func (c *TreeClient) getVersions(ctx context.Context, cnrID cid.ID, treeID, filepath string, onlyUnversioned bool) ([]*data.NodeVersion, error) { - keysToReturn := []string{oidKV, isUnversionedKV, isDeleteMarkerKV} + keysToReturn := []string{oidKV, isUnversionedKV, isDeleteMarkerKV, etagKV, sizeKV} path := pathFromName(filepath) p := &getNodesParams{ CnrID: cnrID,