diff --git a/CHANGELOG.md b/CHANGELOG.md
index 00451ea..a8aca82 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -25,6 +25,7 @@ This document outlines major changes between releases.
- Support multiple version credentials using GSet (#135)
- Implement chunk uploading (#106)
- Add new `kludge.bypass_content_encoding_check_in_chunks` config param (#146)
+- Add new `frostfs.client_cut` config param (#192)
### Changed
- Update prometheus to v1.15.0 (#94)
diff --git a/api/handler/attributes_test.go b/api/handler/attributes_test.go
index 23ca8ac..2889d73 100644
--- a/api/handler/attributes_test.go
+++ b/api/handler/attributes_test.go
@@ -17,7 +17,7 @@ func TestGetObjectPartsAttributes(t *testing.T) {
createTestBucket(hc, bktName)
- putObject(t, hc, bktName, objName)
+ putObject(hc, bktName, objName)
result := getObjectAttributes(hc, bktName, objName, objectParts)
require.Nil(t, result.ObjectParts)
diff --git a/api/handler/delete_test.go b/api/handler/delete_test.go
index f084e46..6c44ec3 100644
--- a/api/handler/delete_test.go
+++ b/api/handler/delete_test.go
@@ -25,7 +25,7 @@ func TestDeleteBucketOnAlreadyRemovedError(t *testing.T) {
bktName, objName := "bucket-for-removal", "object-to-delete"
bktInfo := createTestBucket(hc, bktName)
- putObject(t, hc, bktName, objName)
+ putObject(hc, bktName, objName)
addr := getAddressOfLastVersion(hc, bktInfo, objName)
hc.tp.SetObjectError(addr, &apistatus.ObjectAlreadyRemoved{})
@@ -66,7 +66,7 @@ func TestDeleteBucketOnNotFoundError(t *testing.T) {
bktName, objName := "bucket-for-removal", "object-to-delete"
bktInfo := createTestBucket(hc, bktName)
- putObject(t, hc, bktName, objName)
+ putObject(hc, bktName, objName)
nodeVersion, err := hc.tree.GetUnversioned(hc.context, bktInfo, objName)
require.NoError(t, err)
@@ -98,7 +98,7 @@ func TestDeleteObjectFromSuspended(t *testing.T) {
bktName, objName := "bucket-versioned-for-removal", "object-to-delete"
createSuspendedBucket(t, tc, bktName)
- putObject(t, tc, bktName, objName)
+ putObject(tc, bktName, objName)
versionID, isDeleteMarker := deleteObject(t, tc, bktName, objName, emptyVersion)
require.True(t, isDeleteMarker)
@@ -255,7 +255,7 @@ func TestDeleteMarkerSuspended(t *testing.T) {
t.Run("remove last unversioned non delete marker", func(t *testing.T) {
objName := "obj3"
- putObject(t, tc, bktName, objName)
+ putObject(tc, bktName, objName)
nodeVersion, err := tc.tree.GetUnversioned(tc.Context(), bktInfo, objName)
require.NoError(t, err)
@@ -475,11 +475,11 @@ func getVersion(resp *ListObjectsVersionsResponse, objName string) []*ObjectVers
return res
}
-func putObject(t *testing.T, tc *handlerContext, bktName, objName string) {
+func putObject(hc *handlerContext, bktName, objName string) {
body := bytes.NewReader([]byte("content"))
- w, r := prepareTestPayloadRequest(tc, bktName, objName, body)
- tc.Handler().PutObjectHandler(w, r)
- assertStatus(t, w, http.StatusOK)
+ w, r := prepareTestPayloadRequest(hc, bktName, objName, body)
+ hc.Handler().PutObjectHandler(w, r)
+ assertStatus(hc.t, w, http.StatusOK)
}
func createSuspendedBucket(t *testing.T, tc *handlerContext, bktName string) *data.BucketInfo {
diff --git a/api/handler/get_test.go b/api/handler/get_test.go
index 6a480ef..78c0fbc 100644
--- a/api/handler/get_test.go
+++ b/api/handler/get_test.go
@@ -186,7 +186,7 @@ func TestGetObject(t *testing.T) {
bktName, objName := "bucket", "obj"
bktInfo, objInfo := createVersionedBucketAndObject(hc.t, hc, bktName, objName)
- putObject(hc.t, hc, bktName, objName)
+ putObject(hc, bktName, objName)
checkFound(hc.t, hc, bktName, objName, objInfo.VersionID())
checkFound(hc.t, hc, bktName, objName, emptyVersion)
diff --git a/api/handler/handlers_test.go b/api/handler/handlers_test.go
index 91d96f8..a6d6b07 100644
--- a/api/handler/handlers_test.go
+++ b/api/handler/handlers_test.go
@@ -38,6 +38,8 @@ type handlerContext struct {
tree *tree.Tree
context context.Context
kludge *kludgeSettingsMock
+
+ layerFeatures *layer.FeatureSettingsMock
}
func (hc *handlerContext) Handler() *handler {
@@ -123,11 +125,14 @@ func prepareHandlerContextBase(t *testing.T, minCache bool) *handlerContext {
cacheCfg = getMinCacheConfig(l)
}
+ features := &layer.FeatureSettingsMock{}
+
layerCfg := &layer.Config{
Caches: cacheCfg,
AnonKey: layer.AnonymousKey{Key: key},
Resolver: testResolver,
TreeService: treeMock,
+ Features: features,
}
var pp netmap.PlacementPolicy
@@ -154,6 +159,8 @@ func prepareHandlerContextBase(t *testing.T, minCache bool) *handlerContext {
tree: treeMock,
context: middleware.SetBoxData(context.Background(), newTestAccessBox(t, key)),
kludge: kludge,
+
+ layerFeatures: features,
}
}
diff --git a/api/handler/head_test.go b/api/handler/head_test.go
index e2c943c..821f651 100644
--- a/api/handler/head_test.go
+++ b/api/handler/head_test.go
@@ -114,7 +114,7 @@ func TestHeadObject(t *testing.T) {
bktName, objName := "bucket", "obj"
bktInfo, objInfo := createVersionedBucketAndObject(hc.t, hc, bktName, objName)
- putObject(hc.t, hc, bktName, objName)
+ putObject(hc, bktName, objName)
checkFound(hc.t, hc, bktName, objName, objInfo.VersionID())
checkFound(hc.t, hc, bktName, objName, emptyVersion)
diff --git a/api/handler/locking_test.go b/api/handler/locking_test.go
index 0e1986d..7e415a4 100644
--- a/api/handler/locking_test.go
+++ b/api/handler/locking_test.go
@@ -536,7 +536,7 @@ func TestPutObjectWithLock(t *testing.T) {
createTestBucketWithLock(hc, bktName, lockConfig)
objDefault := "obj-default-retention"
- putObject(t, hc, bktName, objDefault)
+ putObject(hc, bktName, objDefault)
getObjectRetentionApproximate(hc, bktName, objDefault, governanceMode, time.Now().Add(24*time.Hour))
getObjectLegalHold(hc, bktName, objDefault, legalHoldOff)
@@ -587,7 +587,7 @@ func TestPutLockErrors(t *testing.T) {
headers[api.AmzObjectLockRetainUntilDate] = "dummy"
putObjectWithLockFailed(t, hc, bktName, objName, headers, apiErrors.ErrInvalidRetentionDate)
- putObject(t, hc, bktName, objName)
+ putObject(hc, bktName, objName)
retention := &data.Retention{Mode: governanceMode}
putObjectRetentionFailed(t, hc, bktName, objName, retention, apiErrors.ErrMalformedXML)
diff --git a/api/handler/put_test.go b/api/handler/put_test.go
index c07c270..1e21047 100644
--- a/api/handler/put_test.go
+++ b/api/handler/put_test.go
@@ -23,6 +23,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
+ "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/stretchr/testify/require"
)
@@ -309,3 +310,37 @@ func TestCreateBucket(t *testing.T) {
box2, _ := createAccessBox(t)
createBucketAssertS3Error(hc, bktName, box2, s3errors.ErrBucketAlreadyExists)
}
+
+func TestPutObjectClientCut(t *testing.T) {
+ hc := prepareHandlerContext(t)
+ bktName, objName1, objName2 := "bkt-name", "obj-name1", "obj-name2"
+ createTestBucket(hc, bktName)
+
+ putObject(hc, bktName, objName1)
+ obj1 := getObjectFromLayer(hc, objName1)[0]
+ require.Empty(t, getObjectAttribute(obj1, "s3-client-cut"))
+
+ hc.layerFeatures.SetClientCut(true)
+ putObject(hc, bktName, objName2)
+ obj2 := getObjectFromLayer(hc, objName2)[0]
+ require.Equal(t, "true", getObjectAttribute(obj2, "s3-client-cut"))
+}
+
+func getObjectFromLayer(hc *handlerContext, objName string) []*object.Object {
+ var res []*object.Object
+ for _, o := range hc.tp.Objects() {
+ if objName == getObjectAttribute(o, object.AttributeFilePath) {
+ res = append(res, o)
+ }
+ }
+ return res
+}
+
+func getObjectAttribute(obj *object.Object, attrName string) string {
+ for _, attr := range obj.Attributes() {
+ if attr.Key() == attrName {
+ return attr.Value()
+ }
+ }
+ return ""
+}
diff --git a/api/layer/frostfs.go b/api/layer/frostfs.go
index b70f55f..26f7352 100644
--- a/api/layer/frostfs.go
+++ b/api/layer/frostfs.go
@@ -111,6 +111,9 @@ type PrmObjectCreate struct {
// Number of object copies that is enough to consider put successful.
CopiesNumber []uint32
+
+ // Enables client side object preparing.
+ ClientCut bool
}
// PrmObjectDelete groups parameters of FrostFS.DeleteObject operation.
diff --git a/api/layer/frostfs_mock.go b/api/layer/frostfs_mock.go
index e550cd8..32eac88 100644
--- a/api/layer/frostfs_mock.go
+++ b/api/layer/frostfs_mock.go
@@ -25,6 +25,18 @@ import (
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
)
+type FeatureSettingsMock struct {
+ clientCut bool
+}
+
+func (k *FeatureSettingsMock) ClientCut() bool {
+ return k.clientCut
+}
+
+func (k *FeatureSettingsMock) SetClientCut(clientCut bool) {
+ k.clientCut = clientCut
+}
+
type TestFrostFS struct {
FrostFS
@@ -222,6 +234,13 @@ func (t *TestFrostFS) CreateObject(_ context.Context, prm PrmObjectCreate) (oid.
attrs = append(attrs, *a)
}
+ if prm.ClientCut {
+ a := object.NewAttribute()
+ a.SetKey("s3-client-cut")
+ a.SetValue("true")
+ attrs = append(attrs, *a)
+ }
+
for i := range prm.Attributes {
a := object.NewAttribute()
a.SetKey(prm.Attributes[i][0])
diff --git a/api/layer/layer.go b/api/layer/layer.go
index 3fbe985..91a3d83 100644
--- a/api/layer/layer.go
+++ b/api/layer/layer.go
@@ -45,6 +45,10 @@ type (
Resolve(ctx context.Context, name string) (cid.ID, error)
}
+ FeatureSettings interface {
+ ClientCut() bool
+ }
+
layer struct {
frostFS FrostFS
gateOwner user.ID
@@ -54,6 +58,7 @@ type (
ncontroller EventListener
cache *Cache
treeService TreeService
+ features FeatureSettings
}
Config struct {
@@ -63,6 +68,7 @@ type (
AnonKey AnonymousKey
Resolver BucketResolver
TreeService TreeService
+ Features FeatureSettings
}
// AnonymousKey contains data for anonymous requests.
@@ -301,6 +307,7 @@ func NewLayer(log *zap.Logger, frostFS FrostFS, config *Config) Client {
resolver: config.Resolver,
cache: NewCache(config.Caches),
treeService: config.TreeService,
+ features: config.Features,
}
}
diff --git a/api/layer/object.go b/api/layer/object.go
index 9cda57b..09a5701 100644
--- a/api/layer/object.go
+++ b/api/layer/object.go
@@ -458,6 +458,7 @@ func (n *layer) objectDelete(ctx context.Context, bktInfo *data.BucketInfo, idOb
// Returns object ID and payload sha256 hash.
func (n *layer) objectPutAndHash(ctx context.Context, prm PrmObjectCreate, bktInfo *data.BucketInfo) (uint64, oid.ID, []byte, error) {
n.prepareAuthParameters(ctx, &prm.PrmAuth, bktInfo.Owner)
+ prm.ClientCut = n.features.ClientCut()
var size uint64
hash := sha256.New()
prm.Payload = wrapReader(prm.Payload, 64*1024, func(buf []byte) {
diff --git a/api/layer/versioning_test.go b/api/layer/versioning_test.go
index 897c988..4476978 100644
--- a/api/layer/versioning_test.go
+++ b/api/layer/versioning_test.go
@@ -170,6 +170,7 @@ func prepareContext(t *testing.T, cachesConfig ...*CachesConfig) *testContext {
Caches: config,
AnonKey: AnonymousKey{Key: key},
TreeService: NewTreeService(),
+ Features: &FeatureSettingsMock{},
}
return &testContext{
diff --git a/cmd/s3-gw/app.go b/cmd/s3-gw/app.go
index 272af05..a0db052 100644
--- a/cmd/s3-gw/app.go
+++ b/cmd/s3-gw/app.go
@@ -71,6 +71,7 @@ type (
xmlDecoder *xml.DecoderProvider
maxClient maxClientsConfig
bypassContentEncodingInChunks atomic.Bool
+ clientCut atomic.Bool
}
maxClientsConfig struct {
@@ -144,6 +145,7 @@ func (a *App) initLayer(ctx context.Context) {
GateOwner: gateOwner,
Resolver: a.bucketResolver,
TreeService: tree.NewTree(services.NewPoolWrapper(a.treePool), a.log),
+ Features: a.settings,
}
// prepare object layer
@@ -176,6 +178,7 @@ func newAppSettings(log *Logger, v *viper.Viper) *appSettings {
}
settings.setBypassContentEncodingInChunks(v.GetBool(cfgKludgeBypassContentEncodingCheckInChunks))
+ settings.setClientCut(v.GetBool(cfgClientCut))
return settings
}
@@ -188,6 +191,14 @@ func (s *appSettings) setBypassContentEncodingInChunks(bypass bool) {
s.bypassContentEncodingInChunks.Store(bypass)
}
+func (s *appSettings) ClientCut() bool {
+ return s.clientCut.Load()
+}
+
+func (s *appSettings) setClientCut(clientCut bool) {
+ s.clientCut.Store(clientCut)
+}
+
func (a *App) initAPI(ctx context.Context) {
a.initLayer(ctx)
a.initHandler()
@@ -568,6 +579,7 @@ func (a *App) updateSettings() {
a.settings.xmlDecoder.UseDefaultNamespaceForCompleteMultipart(a.cfg.GetBool(cfgKludgeUseDefaultXMLNSForCompleteMultipartUpload))
a.settings.setBypassContentEncodingInChunks(a.cfg.GetBool(cfgKludgeBypassContentEncodingCheckInChunks))
+ a.settings.setClientCut(a.cfg.GetBool(cfgClientCut))
}
func (a *App) startServices() {
diff --git a/cmd/s3-gw/app_settings.go b/cmd/s3-gw/app_settings.go
index db8a193..ce20905 100644
--- a/cmd/s3-gw/app_settings.go
+++ b/cmd/s3-gw/app_settings.go
@@ -138,6 +138,8 @@ const ( // Settings.
// Configuration of parameters of requests to FrostFS.
// Number of the object copies to consider PUT to FrostFS successful.
cfgSetCopiesNumber = "frostfs.set_copies_number"
+ // Enabling client side object preparing for PUT operations.
+ cfgClientCut = "frostfs.client_cut"
// List of allowed AccessKeyID prefixes.
cfgAllowedAccessKeyIDPrefixes = "allowed_access_key_id_prefixes"
diff --git a/config/config.env b/config/config.env
index 0bb2459..a1643d1 100644
--- a/config/config.env
+++ b/config/config.env
@@ -125,6 +125,8 @@ S3_GW_CORS_DEFAULT_MAX_AGE=600
# to consider PUT to FrostFS successful.
# `0` or empty list means that object will be processed according to the container's placement policy
S3_GW_FROSTFS_SET_COPIES_NUMBER=0
+# This flag enables client side object preparing.
+S3_GW_FROSTFS_CLIENT_CUT=false
# List of allowed AccessKeyID prefixes
# If not set, S3 GW will accept all AccessKeyIDs
diff --git a/config/config.yaml b/config/config.yaml
index 5924918..e226bdc 100644
--- a/config/config.yaml
+++ b/config/config.yaml
@@ -150,6 +150,8 @@ frostfs:
# Numbers of the object copies (for each replica) to consider PUT to FrostFS successful.
# `[0]` or empty list means that object will be processed according to the container's placement policy
set_copies_number: [0]
+ # This flag enables client side object preparing.
+ client_cut: false
# List of allowed AccessKeyID prefixes
# If the parameter is omitted, S3 GW will accept all AccessKeyIDs
diff --git a/docs/configuration.md b/docs/configuration.md
index 9d744b5..50eaa48 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -500,17 +500,20 @@ tracing:
# `frostfs` section
Contains parameters of requests to FrostFS.
-This value can be overridden with `X-Amz-Meta-Frostfs-Copies-Number` (value is comma separated numbers: `1,2,3`)
+
+The `set_copies_number` value can be overridden with `X-Amz-Meta-Frostfs-Copies-Number` (value is comma separated numbers: `1,2,3`)
header for `PutObject`, `CopyObject`, `CreateMultipartUpload`.
```yaml
frostfs:
set_copies_number: [0]
+ client_cut: false
```
-| Parameter | Type | Default value | Description |
-|---------------------|------------|---------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| `set_copies_number` | `[]uint32` | `[0]` | Numbers of the object copies (for each replica) to consider PUT to FrostFS successful.
Default value `[0]` or empty list means that object will be processed according to the container's placement policy |
+| Parameter | Type | SIGHUP reload | Default value | Description |
+|---------------------|------------|---------------|---------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `set_copies_number` | `[]uint32` | yes | `[0]` | Numbers of the object copies (for each replica) to consider PUT to FrostFS successful.
Default value `[0]` or empty list means that object will be processed according to the container's placement policy |
+| `client_cut` | `bool` | yes | `false` | This flag enables client side object preparing. |
# `resolve_bucket` section
diff --git a/go.sum b/go.sum
index 4950161..b1099ce 100644
--- a/go.sum
+++ b/go.sum
@@ -44,8 +44,6 @@ git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSV
git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU=
git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 h1:aGQ6QaAnTerQ5Dq5b2/f9DUQtSqPkZZ/bkMx/HKuLCo=
git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6/go.mod h1:W8Nn08/l6aQ7UlIbpF7FsQou7TVpcRD1ZT1KG4TrFhE=
-git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230821073319-342524159ac3 h1:GBRTOTRrtIvxi2TgxG7z/J7uRXiyb1SxR4247FaYCgU=
-git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230821073319-342524159ac3/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw=
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230821090303-202412230a05 h1:OuViMF54N87FXmaBEpYw3jhzaLrJ/EWOlPL1wUkimE0=
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230821090303-202412230a05/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw=
git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc=
diff --git a/internal/frostfs/frostfs.go b/internal/frostfs/frostfs.go
index 8b69c18..29e2d42 100644
--- a/internal/frostfs/frostfs.go
+++ b/internal/frostfs/frostfs.go
@@ -243,6 +243,7 @@ func (x *FrostFS) CreateObject(ctx context.Context, prm layer.PrmObjectCreate) (
prmPut.SetHeader(*obj)
prmPut.SetPayload(prm.Payload)
prmPut.SetCopiesNumberVector(prm.CopiesNumber)
+ prmPut.SetClientCut(prm.ClientCut)
if prm.BearerToken != nil {
prmPut.UseBearer(*prm.BearerToken)