[#192] Support client cut

Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>

CHANGELOG.md

@@ -25,6 +25,7 @@ This document outlines major changes between releases.
 - Support multiple version credentials using GSet (#135)
 - Implement chunk uploading (#106)
 - Add new `kludge.bypass_content_encoding_check_in_chunks` config param (#146)
+- Add new `frostfs.client_cut` config param (#192)
 
 ### Changed
 - Update prometheus to v1.15.0 (#94)

api/layer/frostfs.go

@@ -111,6 +111,9 @@ type PrmObjectCreate struct {
 
 	// Number of object copies that is enough to consider put successful.
 	CopiesNumber []uint32
+
+	// Enables client side object preparing.
+	ClientCut bool
 }
 
 // PrmObjectDelete groups parameters of FrostFS.DeleteObject operation.

api/layer/layer.go

@@ -45,6 +45,10 @@ type (
 		Resolve(ctx context.Context, name string) (cid.ID, error)
 	}
 
+	FeatureSettings interface {
+		ClientCut() bool
+	}
+
 	layer struct {
 		frostFS     FrostFS
 		gateOwner   user.ID
@@ -54,6 +58,7 @@ type (
 		ncontroller EventListener
 		cache       *Cache
 		treeService TreeService
+		features    FeatureSettings
 	}
 
 	Config struct {
@@ -63,6 +68,7 @@ type (
 		AnonKey      AnonymousKey
 		Resolver     BucketResolver
 		TreeService  TreeService
+		Features     FeatureSettings
 	}
 
 	// AnonymousKey contains data for anonymous requests.
@@ -301,6 +307,7 @@ func NewLayer(log *zap.Logger, frostFS FrostFS, config *Config) Client {
 		resolver:    config.Resolver,
 		cache:       NewCache(config.Caches),
 		treeService: config.TreeService,
+		features:    config.Features,
 	}
 }
 

api/layer/object.go

@@ -458,6 +458,7 @@ func (n *layer) objectDelete(ctx context.Context, bktInfo *data.BucketInfo, idOb
 // Returns object ID and payload sha256 hash.
 func (n *layer) objectPutAndHash(ctx context.Context, prm PrmObjectCreate, bktInfo *data.BucketInfo) (uint64, oid.ID, []byte, error) {
 	n.prepareAuthParameters(ctx, &prm.PrmAuth, bktInfo.Owner)
+	prm.ClientCut = n.features.ClientCut()
 	var size uint64
 	hash := sha256.New()
 	prm.Payload = wrapReader(prm.Payload, 64*1024, func(buf []byte) {

cmd/s3-gw/app.go

@@ -71,6 +71,7 @@ type (
 		xmlDecoder                    *xml.DecoderProvider
 		maxClient                     maxClientsConfig
 		bypassContentEncodingInChunks atomic.Bool
+		clientCut                     atomic.Bool
 	}
 
 	maxClientsConfig struct {
@@ -144,6 +145,7 @@ func (a *App) initLayer(ctx context.Context) {
 		GateOwner:   gateOwner,
 		Resolver:    a.bucketResolver,
 		TreeService: tree.NewTree(services.NewPoolWrapper(a.treePool), a.log),
+		Features:    a.settings,
 	}
 
 	// prepare object layer
@@ -176,6 +178,7 @@ func newAppSettings(log *Logger, v *viper.Viper) *appSettings {
 	}
 
 	settings.setBypassContentEncodingInChunks(v.GetBool(cfgKludgeBypassContentEncodingCheckInChunks))
+	settings.setClientCut(v.GetBool(cfgClientCut))
 
 	return settings
 }
@@ -188,6 +191,14 @@ func (s *appSettings) setBypassContentEncodingInChunks(bypass bool) {
 	s.bypassContentEncodingInChunks.Store(bypass)
 }
 
+func (s *appSettings) ClientCut() bool {
+	return s.clientCut.Load()
+}
+
+func (s *appSettings) setClientCut(clientCut bool) {
+	s.clientCut.Store(clientCut)
+}
+
 func (a *App) initAPI(ctx context.Context) {
 	a.initLayer(ctx)
 	a.initHandler()
@@ -568,6 +579,7 @@ func (a *App) updateSettings() {
 
 	a.settings.xmlDecoder.UseDefaultNamespaceForCompleteMultipart(a.cfg.GetBool(cfgKludgeUseDefaultXMLNSForCompleteMultipartUpload))
 	a.settings.setBypassContentEncodingInChunks(a.cfg.GetBool(cfgKludgeBypassContentEncodingCheckInChunks))
+	a.settings.setClientCut(a.cfg.GetBool(cfgClientCut))
 }
 
 func (a *App) startServices() {

cmd/s3-gw/app_settings.go

@@ -138,6 +138,8 @@ const ( // Settings.
 	// Configuration of parameters of requests to FrostFS.
 	// Number of the object copies to consider PUT to FrostFS successful.
 	cfgSetCopiesNumber = "frostfs.set_copies_number"
+	// Enabling client side object preparing for PUT operations.
+	cfgClientCut = "frostfs.client_cut"
 
 	// List of allowed AccessKeyID prefixes.
 	cfgAllowedAccessKeyIDPrefixes = "allowed_access_key_id_prefixes"

config/config.env

@@ -125,6 +125,8 @@ S3_GW_CORS_DEFAULT_MAX_AGE=600
 # to consider PUT to FrostFS successful.
 # `0` or empty list means that object will be processed according to the container's placement policy
 S3_GW_FROSTFS_SET_COPIES_NUMBER=0
+# This flag enables client side object preparing.
+S3_GW_FROSTFS_CLIENT_CUT=false
 
 # List of allowed AccessKeyID prefixes
 # If not set, S3 GW will accept all AccessKeyIDs

config/config.yaml

@@ -150,6 +150,8 @@ frostfs:
   # Numbers of the object copies (for each replica) to consider PUT to FrostFS successful.
   # `[0]` or empty list means that object will be processed according to the container's placement policy
   set_copies_number: [0]
+  # This flag enables client side object preparing.
+  client_cut: false
 
 # List of allowed AccessKeyID prefixes
 # If the parameter is omitted, S3 GW will accept all AccessKeyIDs

docs/configuration.md

@@ -500,17 +500,20 @@ tracing:
 # `frostfs` section
 
 Contains parameters of requests to FrostFS.
-This value can be overridden with `X-Amz-Meta-Frostfs-Copies-Number` (value is comma separated numbers: `1,2,3`)
+
+The `set_copies_number` value can be overridden with `X-Amz-Meta-Frostfs-Copies-Number` (value is comma separated numbers: `1,2,3`)
 header for `PutObject`, `CopyObject`, `CreateMultipartUpload`.
 
 ```yaml
 frostfs:
   set_copies_number: [0]
+  client_cut: false
 ```
 
-| Parameter           | Type       | Default value | Description                                                                                                                                                                                                     |
+| Parameter           | Type       | SIGHUP reload | Default value | Description                                                                                                                                                                                                     |
-|---------------------|------------|---------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+|---------------------|------------|---------------|---------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| `set_copies_number` | `[]uint32` | `[0]`         | Numbers of the object copies (for each replica) to consider PUT to FrostFS successful. <br/>Default value `[0]` or empty list means that object will be processed according to the container's placement policy |
+| `set_copies_number` | `[]uint32` | yes           | `[0]`         | Numbers of the object copies (for each replica) to consider PUT to FrostFS successful. <br/>Default value `[0]` or empty list means that object will be processed according to the container's placement policy |
+| `client_cut`        | `bool`     | yes           | `false`       | This flag enables client side object preparing.                                                                                                                                                                 |
 
 # `resolve_bucket` section
 

go.sum

@@ -44,8 +44,6 @@ git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSV
 git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU=
 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 h1:aGQ6QaAnTerQ5Dq5b2/f9DUQtSqPkZZ/bkMx/HKuLCo=
 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6/go.mod h1:W8Nn08/l6aQ7UlIbpF7FsQou7TVpcRD1ZT1KG4TrFhE=
-git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230821073319-342524159ac3 h1:GBRTOTRrtIvxi2TgxG7z/J7uRXiyb1SxR4247FaYCgU=
-git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230821073319-342524159ac3/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw=
 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230821090303-202412230a05 h1:OuViMF54N87FXmaBEpYw3jhzaLrJ/EWOlPL1wUkimE0=
 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230821090303-202412230a05/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw=
 git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc=

internal/frostfs/frostfs.go

@@ -243,6 +243,7 @@ func (x *FrostFS) CreateObject(ctx context.Context, prm layer.PrmObjectCreate) (
 	prmPut.SetHeader(*obj)
 	prmPut.SetPayload(prm.Payload)
 	prmPut.SetCopiesNumberVector(prm.CopiesNumber)
+	prmPut.SetClientCut(prm.ClientCut)
 
 	if prm.BearerToken != nil {
 		prmPut.UseBearer(*prm.BearerToken)