[#634] Add CopiesNumber in NeoFS requests

Signed-off-by: Angira Kekteeva <kira@nspcc.ru>

api/handler/api.go

@@ -28,11 +28,16 @@ type (
 		DefaultMaxAge      int
 		NotificatorEnabled bool
 		TLSEnabled         bool
+		CopiesNumber       uint32
 	}
 )
 
-// DefaultPolicy is a default policy of placing containers in NeoFS if it's not set at the request.
+const (
-const DefaultPolicy = "REP 3"
+	// DefaultPolicy is a default policy of placing containers in NeoFS if it's not set at the request.
+	DefaultPolicy = "REP 3"
+	// DefaultCopiesNumber is a default number of object copies that is enough to consider put successful if it's not set in config.
+	DefaultCopiesNumber uint32 = 0
+)
 
 var _ api.Handler = (*handler)(nil)
 

api/handler/cors.go

@@ -48,8 +48,9 @@ func (h *handler) PutBucketCorsHandler(w http.ResponseWriter, r *http.Request) {
 	}
 
 	p := &layer.PutCORSParams{
-		BktInfo: bktInfo,
+		BktInfo:      bktInfo,
-		Reader:  r.Body,
+		Reader:       r.Body,
+		CopiesNumber: h.cfg.CopiesNumber,
 	}
 
 	if err = h.obj.PutBucketCORS(r.Context(), p); err != nil {

api/handler/locking.go

@@ -129,18 +129,21 @@ func (h *handler) PutObjectLegalHoldHandler(w http.ResponseWriter, r *http.Reque
 		return
 	}
 
-	p := &layer.ObjectVersion{
+	p := &layer.PutLockInfoParams{
-		BktInfo:    bktInfo,
+		ObjVersion: &layer.ObjectVersion{
-		ObjectName: reqInfo.ObjectName,
+			BktInfo:    bktInfo,
-		VersionID:  reqInfo.URL.Query().Get(api.QueryVersionID),
+			ObjectName: reqInfo.ObjectName,
-	}
+			VersionID:  reqInfo.URL.Query().Get(api.QueryVersionID),
-	lock := &data.ObjectLock{
-		LegalHold: &data.LegalHoldLock{
-			Enabled: legalHold.Status == legalHoldOn,
 		},
+		NewLock: &data.ObjectLock{
+			LegalHold: &data.LegalHoldLock{
+				Enabled: legalHold.Status == legalHoldOn,
+			},
+		},
+		CopiesNumber: h.cfg.CopiesNumber,
 	}
 
-	if err = h.obj.PutLockInfo(r.Context(), p, lock); err != nil {
+	if err = h.obj.PutLockInfo(r.Context(), p); err != nil {
 		h.logAndSendError(w, "couldn't head put legal hold", reqInfo, err)
 		return
 	}
@@ -209,13 +212,17 @@ func (h *handler) PutObjectRetentionHandler(w http.ResponseWriter, r *http.Reque
 		return
 	}
 
-	p := &layer.ObjectVersion{
+	p := &layer.PutLockInfoParams{
-		BktInfo:    bktInfo,
+		ObjVersion: &layer.ObjectVersion{
-		ObjectName: reqInfo.ObjectName,
+			BktInfo:    bktInfo,
-		VersionID:  reqInfo.URL.Query().Get(api.QueryVersionID),
+			ObjectName: reqInfo.ObjectName,
+			VersionID:  reqInfo.URL.Query().Get(api.QueryVersionID),
+		},
+		NewLock:      lock,
+		CopiesNumber: h.cfg.CopiesNumber,
 	}
 
-	if err = h.obj.PutLockInfo(r.Context(), p, lock); err != nil {
+	if err = h.obj.PutLockInfo(r.Context(), p); err != nil {
 		h.logAndSendError(w, "couldn't put legal hold", reqInfo, err)
 		return
 	}

api/handler/multipart_upload.go

@@ -215,9 +215,10 @@ func (h *handler) UploadPartHandler(w http.ResponseWriter, r *http.Request) {
 			Bkt:      bktInfo,
 			Key:      reqInfo.ObjectName,
 		},
-		PartNumber: partNumber,
+		PartNumber:   partNumber,
-		Size:       r.ContentLength,
+		Size:         r.ContentLength,
-		Reader:     r.Body,
+		Reader:       r.Body,
+		CopiesNumber: h.cfg.CopiesNumber,
 	}
 
 	p.Info.Encryption, err = h.formEncryptionParams(r.Header)
@@ -315,10 +316,11 @@ func (h *handler) UploadPartCopy(w http.ResponseWriter, r *http.Request) {
 			Bkt:      bktInfo,
 			Key:      reqInfo.ObjectName,
 		},
-		SrcObjInfo: srcInfo,
+		SrcObjInfo:   srcInfo,
-		SrcBktInfo: srcBktInfo,
+		SrcBktInfo:   srcBktInfo,
-		PartNumber: partNumber,
+		PartNumber:   partNumber,
-		Range:      srcRange,
+		Range:        srcRange,
+		CopiesNumber: h.cfg.CopiesNumber,
 	}
 
 	p.Info.Encryption, err = h.formEncryptionParams(r.Header)

api/handler/notifications.go

@@ -116,6 +116,7 @@ func (h *handler) PutBucketNotificationHandler(w http.ResponseWriter, r *http.Re
 		RequestInfo:   reqInfo,
 		BktInfo:       bktInfo,
 		Configuration: conf,
+		CopiesNumber:  h.cfg.CopiesNumber,
 	}
 
 	if err = h.obj.PutBucketNotificationConfiguration(r.Context(), p); err != nil {

api/handler/put.go

@@ -219,12 +219,13 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
 	}
 
 	params := &layer.PutObjectParams{
-		BktInfo:    bktInfo,
+		BktInfo:      bktInfo,
-		Object:     reqInfo.ObjectName,
+		Object:       reqInfo.ObjectName,
-		Reader:     r.Body,
+		Reader:       r.Body,
-		Size:       r.ContentLength,
+		Size:         r.ContentLength,
-		Header:     metadata,
+		Header:       metadata,
-		Encryption: encryption,
+		Encryption:   encryption,
+		CopiesNumber: h.cfg.CopiesNumber,
 	}
 
 	settings, err := h.obj.GetBucketSettings(r.Context(), bktInfo)

api/layer/cors.go

@@ -37,10 +37,11 @@ func (n *layer) PutBucketCORS(ctx context.Context, p *PutCORSParams) error {
 	}
 
 	prm := PrmObjectCreate{
-		Container: p.BktInfo.CID,
+		Container:    p.BktInfo.CID,
-		Creator:   p.BktInfo.Owner,
+		Creator:      p.BktInfo.Owner,
-		Payload:   p.Reader,
+		Payload:      p.Reader,
-		Filename:  p.BktInfo.CORSObjectName(),
+		Filename:     p.BktInfo.CORSObjectName(),
+		CopiesNumber: p.CopiesNumber,
 	}
 
 	objID, _, err := n.objectPutAndHash(ctx, prm, p.BktInfo)

api/layer/layer.go

@@ -110,13 +110,14 @@ type (
 
 	// PutObjectParams stores object put request parameters.
 	PutObjectParams struct {
-		BktInfo    *data.BucketInfo
+		BktInfo      *data.BucketInfo
-		Object     string
+		Object       string
-		Size       int64
+		Size         int64
-		Reader     io.Reader
+		Reader       io.Reader
-		Header     map[string]string
+		Header       map[string]string
-		Lock       *data.ObjectLock
+		Lock         *data.ObjectLock
-		Encryption encryption.Params
+		Encryption   encryption.Params
+		CopiesNumber uint32
 	}
 
 	DeleteObjectParams struct {
@@ -133,8 +134,9 @@ type (
 
 	// PutCORSParams stores PutCORS request parameters.
 	PutCORSParams struct {
-		BktInfo *data.BucketInfo
+		BktInfo      *data.BucketInfo
-		Reader  io.Reader
+		Reader       io.Reader
+		CopiesNumber uint32
 	}
 
 	// CopyObjectParams stores object copy request parameters.
@@ -215,7 +217,7 @@ type (
 		GetExtendedObjectInfo(ctx context.Context, p *HeadObjectParams) (*data.ExtendedObjectInfo, error)
 
 		GetLockInfo(ctx context.Context, obj *ObjectVersion) (*data.LockInfo, error)
-		PutLockInfo(ctx context.Context, p *ObjectVersion, lock *data.ObjectLock) error
+		PutLockInfo(ctx context.Context, p *PutLockInfoParams) error
 
 		GetBucketTagging(ctx context.Context, cnrID cid.ID) (map[string]string, error)
 		PutBucketTagging(ctx context.Context, cnrID cid.ID, tagSet map[string]string) error

api/layer/multipart_upload.go

@@ -56,18 +56,20 @@ type (
 	}
 
 	UploadPartParams struct {
-		Info       *UploadInfoParams
+		Info         *UploadInfoParams
-		PartNumber int
+		PartNumber   int
-		Size       int64
+		Size         int64
-		Reader     io.Reader
+		Reader       io.Reader
+		CopiesNumber uint32
 	}
 
 	UploadCopyParams struct {
-		Info       *UploadInfoParams
+		Info         *UploadInfoParams
-		SrcObjInfo *data.ObjectInfo
+		SrcObjInfo   *data.ObjectInfo
-		SrcBktInfo *data.BucketInfo
+		SrcBktInfo   *data.BucketInfo
-		PartNumber int
+		PartNumber   int
-		Range      *RangeParams
+		Range        *RangeParams
+		CopiesNumber uint32
 	}
 
 	CompleteMultipartParams struct {
@@ -199,10 +201,11 @@ func (n *layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
 
 	bktInfo := p.Info.Bkt
 	prm := PrmObjectCreate{
-		Container:  bktInfo.CID,
+		Container:    bktInfo.CID,
-		Creator:    bktInfo.Owner,
+		Creator:      bktInfo.Owner,
-		Attributes: make([][2]string, 2),
+		Attributes:   make([][2]string, 2),
-		Payload:    p.Reader,
+		Payload:      p.Reader,
+		CopiesNumber: p.CopiesNumber,
 	}
 
 	decSize := p.Size
@@ -298,10 +301,11 @@ func (n *layer) UploadPartCopy(ctx context.Context, p *UploadCopyParams) (*data.
 	}()
 
 	params := &UploadPartParams{
-		Info:       p.Info,
+		Info:         p.Info,
-		PartNumber: p.PartNumber,
+		PartNumber:   p.PartNumber,
-		Size:       size,
+		Size:         size,
-		Reader:     pr,
+		Reader:       pr,
+		CopiesNumber: p.CopiesNumber,
 	}
 
 	return n.uploadPart(ctx, multipartInfo, params)

api/layer/neofs.go

@@ -121,6 +121,9 @@ type PrmObjectCreate struct {
 
 	// Object payload encapsulated in io.Reader primitive.
 	Payload io.Reader
+
+	// Number of object copies that is enough to consider put successful.
+	CopiesNumber uint32
 }
 
 // PrmObjectDelete groups parameters of NeoFS.DeleteObject operation.

api/layer/notifications.go

@@ -16,6 +16,7 @@ type PutBucketNotificationConfigurationParams struct {
 	RequestInfo   *api.ReqInfo
 	BktInfo       *data.BucketInfo
 	Configuration *data.NotificationConfiguration
+	CopiesNumber  uint32
 }
 
 func (n *layer) PutBucketNotificationConfiguration(ctx context.Context, p *PutBucketNotificationConfigurationParams) error {
@@ -27,10 +28,11 @@ func (n *layer) PutBucketNotificationConfiguration(ctx context.Context, p *PutBu
 	sysName := p.BktInfo.NotificationConfigurationObjectName()
 
 	prm := PrmObjectCreate{
-		Container: p.BktInfo.CID,
+		Container:    p.BktInfo.CID,
-		Creator:   p.BktInfo.Owner,
+		Creator:      p.BktInfo.Owner,
-		Payload:   bytes.NewReader(confXML),
+		Payload:      bytes.NewReader(confXML),
-		Filename:  sysName,
+		Filename:     sysName,
+		CopiesNumber: p.CopiesNumber,
 	}
 
 	objID, _, err := n.objectPutAndHash(ctx, prm, p.BktInfo)

api/layer/object.go

@@ -229,11 +229,12 @@ func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Object
 	}
 
 	prm := PrmObjectCreate{
-		Container:   p.BktInfo.CID,
+		Container:    p.BktInfo.CID,
-		Creator:     own,
+		Creator:      own,
-		PayloadSize: uint64(p.Size),
+		PayloadSize:  uint64(p.Size),
-		Filename:    p.Object,
+		Filename:     p.Object,
-		Payload:     r,
+		Payload:      r,
+		CopiesNumber: p.CopiesNumber,
 	}
 
 	prm.Attributes = make([][2]string, 0, len(p.Header))
@@ -254,13 +255,17 @@ func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Object
 	}
 
 	if p.Lock != nil && (p.Lock.Retention != nil || p.Lock.LegalHold != nil) {
-		objVersion := &ObjectVersion{
+		prm := &PutLockInfoParams{
-			BktInfo:    p.BktInfo,
+			ObjVersion: &ObjectVersion{
-			ObjectName: p.Object,
+				BktInfo:    p.BktInfo,
-			VersionID:  id.EncodeToString(),
+				ObjectName: p.Object,
+				VersionID:  id.EncodeToString(),
+			},
+			NewLock:      p.Lock,
+			CopiesNumber: p.CopiesNumber,
 		}
 
-		if err = n.PutLockInfo(ctx, objVersion, p.Lock); err != nil {
+		if err = n.PutLockInfo(ctx, prm); err != nil {
 			return nil, err
 		}
 	}

api/layer/system_object.go

@@ -19,9 +19,18 @@ const (
 	AttributeExpirationEpoch = "__NEOFS__EXPIRATION_EPOCH"
 )
 
-func (n *layer) PutLockInfo(ctx context.Context, objVersion *ObjectVersion, newLock *data.ObjectLock) error {
+type PutLockInfoParams struct {
-	cnrID := objVersion.BktInfo.CID
+	ObjVersion   *ObjectVersion
-	versionNode, err := n.getNodeVersion(ctx, objVersion)
+	NewLock      *data.ObjectLock
+	CopiesNumber uint32
+}
+
+func (n *layer) PutLockInfo(ctx context.Context, p *PutLockInfoParams) error {
+	var (
+		cnrID   = p.ObjVersion.BktInfo.CID
+		newLock = p.NewLock
+	)
+	versionNode, err := n.getNodeVersion(ctx, p.ObjVersion)
 	if err != nil {
 		return err
 	}
@@ -56,7 +65,7 @@ func (n *layer) PutLockInfo(ctx context.Context, objVersion *ObjectVersion, newL
 			}
 		}
 		lock := &data.ObjectLock{Retention: newLock.Retention}
-		retentionOID, err := n.putLockObject(ctx, objVersion.BktInfo, versionNode.OID, lock)
+		retentionOID, err := n.putLockObject(ctx, p.ObjVersion.BktInfo, versionNode.OID, lock, p.CopiesNumber)
 		if err != nil {
 			return err
 		}
@@ -66,13 +75,13 @@ func (n *layer) PutLockInfo(ctx context.Context, objVersion *ObjectVersion, newL
 	if newLock.LegalHold != nil {
 		if newLock.LegalHold.Enabled && !lockInfo.IsLegalHoldSet() {
 			lock := &data.ObjectLock{LegalHold: newLock.LegalHold}
-			legalHoldOID, err := n.putLockObject(ctx, objVersion.BktInfo, versionNode.OID, lock)
+			legalHoldOID, err := n.putLockObject(ctx, p.ObjVersion.BktInfo, versionNode.OID, lock, p.CopiesNumber)
 			if err != nil {
 				return err
 			}
 			lockInfo.SetLegalHold(legalHoldOID)
 		} else if !newLock.LegalHold.Enabled && lockInfo.IsLegalHoldSet() {
-			if err = n.objectDelete(ctx, objVersion.BktInfo, lockInfo.LegalHold()); err != nil {
+			if err = n.objectDelete(ctx, p.ObjVersion.BktInfo, lockInfo.LegalHold()); err != nil {
 				return fmt.Errorf("couldn't delete lock object '%s' to remove legal hold: %w", lockInfo.LegalHold().EncodeToString(), err)
 			}
 			lockInfo.ResetLegalHold()
@@ -83,18 +92,19 @@ func (n *layer) PutLockInfo(ctx context.Context, objVersion *ObjectVersion, newL
 		return fmt.Errorf("couldn't put lock into tree: %w", err)
 	}
 
-	if err = n.systemCache.PutLockInfo(lockObjectKey(objVersion), lockInfo); err != nil {
+	if err = n.systemCache.PutLockInfo(lockObjectKey(p.ObjVersion), lockInfo); err != nil {
 		n.log.Error("couldn't cache system object", zap.Error(err))
 	}
 
 	return nil
 }
 
-func (n *layer) putLockObject(ctx context.Context, bktInfo *data.BucketInfo, objID oid.ID, lock *data.ObjectLock) (oid.ID, error) {
+func (n *layer) putLockObject(ctx context.Context, bktInfo *data.BucketInfo, objID oid.ID, lock *data.ObjectLock, copiesNumber uint32) (oid.ID, error) {
 	prm := PrmObjectCreate{
-		Container: bktInfo.CID,
+		Container:    bktInfo.CID,
-		Creator:   bktInfo.Owner,
+		Creator:      bktInfo.Owner,
-		Locks:     []oid.ID{objID},
+		Locks:        []oid.ID{objID},
+		CopiesNumber: copiesNumber,
 	}
 
 	var err error

cmd/s3-gw/app.go

@@ -392,10 +392,11 @@ func getAccessBoxCacheConfig(v *viper.Viper, l *zap.Logger) *cache.Config {
 
 func getHandlerOptions(v *viper.Viper, l *zap.Logger) *handler.Config {
 	var (
-		cfg           handler.Config
+		cfg             handler.Config
-		err           error
+		err             error
-		policyStr     = handler.DefaultPolicy
+		policyStr       = handler.DefaultPolicy
-		defaultMaxAge = handler.DefaultMaxAge
+		defaultMaxAge   = handler.DefaultMaxAge
+		setCopiesNumber = handler.DefaultCopiesNumber
 	)
 
 	if v.IsSet(cfgDefaultPolicy) {
@@ -417,9 +418,14 @@ func getHandlerOptions(v *viper.Viper, l *zap.Logger) *handler.Config {
 		}
 	}
 
+	if val := v.GetUint32(cfgSetCopiesNumber); val > 0 {
+		setCopiesNumber = val
+	}
+
 	cfg.DefaultMaxAge = defaultMaxAge
 	cfg.NotificatorEnabled = v.GetBool(cfgEnableNATS)
 	cfg.TLSEnabled = v.IsSet(cfgTLSKeyFile) && v.IsSet(cfgTLSCertFile)
+	cfg.CopiesNumber = setCopiesNumber
 
 	return &cfg
 }

cmd/s3-gw/app_settings.go

@@ -112,6 +112,10 @@ const ( // Settings.
 	cmdPProf   = "pprof"
 	cmdMetrics = "metrics"
 
+	// Configuration of parameters of requests to NeoFS.
+	// Number of the object copies to consider PUT to NeoFS successful.
+	cfgSetCopiesNumber = "neofs.set_copies_number"
+
 	// envPrefix is an environment variables prefix used for configuration.
 	envPrefix = "S3_GW"
 )

internal/neofs/neofs.go

@@ -259,6 +259,7 @@ func (x *NeoFS) CreateObject(ctx context.Context, prm layer.PrmObjectCreate) (oi
 	var prmPut pool.PrmObjectPut
 	prmPut.SetHeader(*obj)
 	prmPut.SetPayload(prm.Payload)
+	prmPut.SetCopiesNumber(prm.CopiesNumber)
 
 	if prm.BearerToken != nil {
 		prmPut.UseBearer(*prm.BearerToken)