[#197] Configure buffer max size for PUT

Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>

api/layer/frostfs.go

@@ -123,6 +123,9 @@ type PrmObjectCreate struct {
 
 	// Disables using Tillich-Zémor hash for payload.
 	WithoutHomomorphicHash bool
+
+	// Sets max buffer size to read payload.
+	BufferMaxSize uint64
 }
 
 // PrmObjectDelete groups parameters of FrostFS.DeleteObject operation.

api/layer/layer.go

@@ -48,6 +48,7 @@ type (
 
 	FeatureSettings interface {
 		ClientCut() bool
+		BufferMaxSizeForPut() uint64
 	}
 
 	layer struct {

api/layer/object.go

@@ -460,6 +460,7 @@ func (n *layer) objectDelete(ctx context.Context, bktInfo *data.BucketInfo, idOb
 func (n *layer) objectPutAndHash(ctx context.Context, prm PrmObjectCreate, bktInfo *data.BucketInfo) (uint64, oid.ID, []byte, error) {
 	n.prepareAuthParameters(ctx, &prm.PrmAuth, bktInfo.Owner)
 	prm.ClientCut = n.features.ClientCut()
+	prm.BufferMaxSize = n.features.BufferMaxSizeForPut()
 	prm.WithoutHomomorphicHash = bktInfo.HomomorphicHashDisabled
 	var size uint64
 	hash := sha256.New()

cmd/s3-gw/app.go

@@ -85,6 +85,7 @@ type (
 		defaultXMLNSForCompleteMultipart bool
 		bypassContentEncodingInChunks    bool
 		clientCut                        bool
+		maxBufferSizeForPut              uint64
 	}
 
 	maxClientsConfig struct {
@@ -189,6 +190,7 @@ func newAppSettings(log *Logger, v *viper.Viper) *appSettings {
 	settings.setBypassContentEncodingInChunks(v.GetBool(cfgKludgeBypassContentEncodingCheckInChunks))
 	settings.setClientCut(v.GetBool(cfgClientCut))
 	settings.initPlacementPolicy(log.logger, v)
+	settings.setBufferMaxSizeForPut(v.GetUint64(cfgBufferMaxSizeForPut))
 
 	return settings
 }
@@ -217,6 +219,18 @@ func (s *appSettings) setClientCut(clientCut bool) {
 	s.mu.Unlock()
 }
 
+func (s *appSettings) BufferMaxSizeForPut() uint64 {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	return s.maxBufferSizeForPut
+}
+
+func (s *appSettings) setBufferMaxSizeForPut(size uint64) {
+	s.mu.Lock()
+	s.maxBufferSizeForPut = size
+	s.mu.Unlock()
+}
+
 func (s *appSettings) initPlacementPolicy(l *zap.Logger, v *viper.Viper) {
 	defaultPolicy := fetchDefaultPolicy(l, v)
 	regionMap := fetchRegionMappingPolicies(l, v)
@@ -589,6 +603,7 @@ func (a *App) updateSettings() {
 	a.settings.useDefaultNamespaceForCompleteMultipart(a.cfg.GetBool(cfgKludgeUseDefaultXMLNSForCompleteMultipartUpload))
 	a.settings.setBypassContentEncodingInChunks(a.cfg.GetBool(cfgKludgeBypassContentEncodingCheckInChunks))
 	a.settings.setClientCut(a.cfg.GetBool(cfgClientCut))
+	a.settings.setBufferMaxSizeForPut(a.cfg.GetUint64(cfgBufferMaxSizeForPut))
 }
 
 func (a *App) startServices() {

cmd/s3-gw/app_settings.go

@@ -149,6 +149,8 @@ const ( // Settings.
 	cfgSetCopiesNumber = "frostfs.set_copies_number"
 	// Enabling client side object preparing for PUT operations.
 	cfgClientCut = "frostfs.client_cut"
+	// Sets max buffer size for read payload in put operations.
+	cfgBufferMaxSizeForPut = "frostfs.buffer_max_size_for_put"
 
 	// List of allowed AccessKeyID prefixes.
 	cfgAllowedAccessKeyIDPrefixes = "allowed_access_key_id_prefixes"
@@ -528,6 +530,9 @@ func newSettings() *viper.Viper {
 	v.SetDefault(cfgPProfAddress, "localhost:8085")
 	v.SetDefault(cfgPrometheusAddress, "localhost:8086")
 
+	// frostfs
+	v.SetDefault(cfgBufferMaxSizeForPut, 1024*1024) // 1mb
+
 	// kludge
 	v.SetDefault(cfgKludgeUseDefaultXMLNSForCompleteMultipartUpload, false)
 	v.SetDefault(cfgKludgeCompleteMultipartUploadKeepalive, 10*time.Second)

config/config.env

@@ -127,6 +127,8 @@ S3_GW_CORS_DEFAULT_MAX_AGE=600
 S3_GW_FROSTFS_SET_COPIES_NUMBER=0
 # This flag enables client side object preparing.
 S3_GW_FROSTFS_CLIENT_CUT=false
+# Sets max buffer size for read payload in put operations.
+S3_GW_FROSTFS_BUFFER_MAX_SIZE_FOR_PUT=1048576
 
 # List of allowed AccessKeyID prefixes
 # If not set, S3 GW will accept all AccessKeyIDs

config/config.yaml

@@ -152,6 +152,8 @@ frostfs:
   set_copies_number: [0]
   # This flag enables client side object preparing.
   client_cut: false
+  # Sets max buffer size for read payload in put operations.
+  buffer_max_size_for_put: 1048576
 
 # List of allowed AccessKeyID prefixes
 # If the parameter is omitted, S3 GW will accept all AccessKeyIDs

docs/configuration.md

@@ -509,12 +509,14 @@ header for `PutObject`, `CopyObject`, `CreateMultipartUpload`.
 frostfs:
   set_copies_number: [0]
   client_cut: false
+  buffer_max_size_for_put: 1048576 # 1mb
 ```
 
 | Parameter                 | Type       | SIGHUP reload | Default value | Description                                                                                                                                                                                                     |
-|---------------------|------------|---------------|---------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+|---------------------------|------------|---------------|---------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
 | `set_copies_number`       | `[]uint32` | yes           | `[0]`         | Numbers of the object copies (for each replica) to consider PUT to FrostFS successful. <br/>Default value `[0]` or empty list means that object will be processed according to the container's placement policy |
 | `client_cut`              | `bool`     | yes           | `false`       | This flag enables client side object preparing.                                                                                                                                                                 |
+| `buffer_max_size_for_put` | `uint64`   | yes           | `1048576`     | Sets max buffer size for read payload in put operations.                                                                                                                                                        |
 
 # `resolve_bucket` section