feature/client-cut #192
11 changed files with 38 additions and 6 deletions
|
@ -25,6 +25,7 @@ This document outlines major changes between releases.
|
||||||
- Support multiple version credentials using GSet (#135)
|
- Support multiple version credentials using GSet (#135)
|
||||||
- Implement chunk uploading (#106)
|
- Implement chunk uploading (#106)
|
||||||
- Add new `kludge.bypass_content_encoding_check_in_chunks` config param (#146)
|
- Add new `kludge.bypass_content_encoding_check_in_chunks` config param (#146)
|
||||||
|
- Add new `frostfs.client_cut` config param (#192)
|
||||||
|
|
||||||
### Changed
|
### Changed
|
||||||
- Update prometheus to v1.15.0 (#94)
|
- Update prometheus to v1.15.0 (#94)
|
||||||
|
|
|
@ -111,6 +111,9 @@ type PrmObjectCreate struct {
|
||||||
|
|
||||||
// Number of object copies that is enough to consider put successful.
|
// Number of object copies that is enough to consider put successful.
|
||||||
CopiesNumber []uint32
|
CopiesNumber []uint32
|
||||||
|
|
||||||
|
// Enables client side object preparing.
|
||||||
|
ClientCut bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// PrmObjectDelete groups parameters of FrostFS.DeleteObject operation.
|
// PrmObjectDelete groups parameters of FrostFS.DeleteObject operation.
|
||||||
|
|
|
@ -45,6 +45,10 @@ type (
|
||||||
Resolve(ctx context.Context, name string) (cid.ID, error)
|
Resolve(ctx context.Context, name string) (cid.ID, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
FeatureSettings interface {
|
||||||
|
ClientCut() bool
|
||||||
|
}
|
||||||
|
|
||||||
layer struct {
|
layer struct {
|
||||||
frostFS FrostFS
|
frostFS FrostFS
|
||||||
gateOwner user.ID
|
gateOwner user.ID
|
||||||
|
@ -54,6 +58,7 @@ type (
|
||||||
ncontroller EventListener
|
ncontroller EventListener
|
||||||
cache *Cache
|
cache *Cache
|
||||||
treeService TreeService
|
treeService TreeService
|
||||||
|
features FeatureSettings
|
||||||
}
|
}
|
||||||
|
|
||||||
Config struct {
|
Config struct {
|
||||||
|
@ -63,6 +68,7 @@ type (
|
||||||
AnonKey AnonymousKey
|
AnonKey AnonymousKey
|
||||||
Resolver BucketResolver
|
Resolver BucketResolver
|
||||||
TreeService TreeService
|
TreeService TreeService
|
||||||
|
Features FeatureSettings
|
||||||
}
|
}
|
||||||
|
|
||||||
// AnonymousKey contains data for anonymous requests.
|
// AnonymousKey contains data for anonymous requests.
|
||||||
|
@ -301,6 +307,7 @@ func NewLayer(log *zap.Logger, frostFS FrostFS, config *Config) Client {
|
||||||
resolver: config.Resolver,
|
resolver: config.Resolver,
|
||||||
cache: NewCache(config.Caches),
|
cache: NewCache(config.Caches),
|
||||||
treeService: config.TreeService,
|
treeService: config.TreeService,
|
||||||
|
features: config.Features,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -458,6 +458,7 @@ func (n *layer) objectDelete(ctx context.Context, bktInfo *data.BucketInfo, idOb
|
||||||
// Returns object ID and payload sha256 hash.
|
// Returns object ID and payload sha256 hash.
|
||||||
func (n *layer) objectPutAndHash(ctx context.Context, prm PrmObjectCreate, bktInfo *data.BucketInfo) (uint64, oid.ID, []byte, error) {
|
func (n *layer) objectPutAndHash(ctx context.Context, prm PrmObjectCreate, bktInfo *data.BucketInfo) (uint64, oid.ID, []byte, error) {
|
||||||
n.prepareAuthParameters(ctx, &prm.PrmAuth, bktInfo.Owner)
|
n.prepareAuthParameters(ctx, &prm.PrmAuth, bktInfo.Owner)
|
||||||
|
prm.ClientCut = n.features.ClientCut()
|
||||||
var size uint64
|
var size uint64
|
||||||
hash := sha256.New()
|
hash := sha256.New()
|
||||||
prm.Payload = wrapReader(prm.Payload, 64*1024, func(buf []byte) {
|
prm.Payload = wrapReader(prm.Payload, 64*1024, func(buf []byte) {
|
||||||
|
|
|
@ -71,6 +71,7 @@ type (
|
||||||
xmlDecoder *xml.DecoderProvider
|
xmlDecoder *xml.DecoderProvider
|
||||||
maxClient maxClientsConfig
|
maxClient maxClientsConfig
|
||||||
bypassContentEncodingInChunks atomic.Bool
|
bypassContentEncodingInChunks atomic.Bool
|
||||||
|
clientCut atomic.Bool
|
||||||
}
|
}
|
||||||
|
|
||||||
maxClientsConfig struct {
|
maxClientsConfig struct {
|
||||||
|
@ -144,6 +145,7 @@ func (a *App) initLayer(ctx context.Context) {
|
||||||
GateOwner: gateOwner,
|
GateOwner: gateOwner,
|
||||||
Resolver: a.bucketResolver,
|
Resolver: a.bucketResolver,
|
||||||
TreeService: tree.NewTree(services.NewPoolWrapper(a.treePool), a.log),
|
TreeService: tree.NewTree(services.NewPoolWrapper(a.treePool), a.log),
|
||||||
|
Features: a.settings,
|
||||||
}
|
}
|
||||||
|
|
||||||
// prepare object layer
|
// prepare object layer
|
||||||
|
@ -176,6 +178,7 @@ func newAppSettings(log *Logger, v *viper.Viper) *appSettings {
|
||||||
}
|
}
|
||||||
|
|
||||||
settings.setBypassContentEncodingInChunks(v.GetBool(cfgKludgeBypassContentEncodingCheckInChunks))
|
settings.setBypassContentEncodingInChunks(v.GetBool(cfgKludgeBypassContentEncodingCheckInChunks))
|
||||||
|
settings.setClientCut(v.GetBool(cfgClientCut))
|
||||||
|
|
||||||
return settings
|
return settings
|
||||||
}
|
}
|
||||||
|
@ -188,6 +191,14 @@ func (s *appSettings) setBypassContentEncodingInChunks(bypass bool) {
|
||||||
s.bypassContentEncodingInChunks.Store(bypass)
|
s.bypassContentEncodingInChunks.Store(bypass)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *appSettings) ClientCut() bool {
|
||||||
|
return s.clientCut.Load()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *appSettings) setClientCut(clientCut bool) {
|
||||||
|
s.clientCut.Store(clientCut)
|
||||||
|
}
|
||||||
|
|
||||||
func (a *App) initAPI(ctx context.Context) {
|
func (a *App) initAPI(ctx context.Context) {
|
||||||
a.initLayer(ctx)
|
a.initLayer(ctx)
|
||||||
a.initHandler()
|
a.initHandler()
|
||||||
|
@ -568,6 +579,7 @@ func (a *App) updateSettings() {
|
||||||
|
|
||||||
a.settings.xmlDecoder.UseDefaultNamespaceForCompleteMultipart(a.cfg.GetBool(cfgKludgeUseDefaultXMLNSForCompleteMultipartUpload))
|
a.settings.xmlDecoder.UseDefaultNamespaceForCompleteMultipart(a.cfg.GetBool(cfgKludgeUseDefaultXMLNSForCompleteMultipartUpload))
|
||||||
a.settings.setBypassContentEncodingInChunks(a.cfg.GetBool(cfgKludgeBypassContentEncodingCheckInChunks))
|
a.settings.setBypassContentEncodingInChunks(a.cfg.GetBool(cfgKludgeBypassContentEncodingCheckInChunks))
|
||||||
|
a.settings.setClientCut(a.cfg.GetBool(cfgClientCut))
|
||||||
}
|
}
|
||||||
|
|
||||||
func (a *App) startServices() {
|
func (a *App) startServices() {
|
||||||
|
|
|
@ -138,6 +138,8 @@ const ( // Settings.
|
||||||
// Configuration of parameters of requests to FrostFS.
|
// Configuration of parameters of requests to FrostFS.
|
||||||
// Number of the object copies to consider PUT to FrostFS successful.
|
// Number of the object copies to consider PUT to FrostFS successful.
|
||||||
cfgSetCopiesNumber = "frostfs.set_copies_number"
|
cfgSetCopiesNumber = "frostfs.set_copies_number"
|
||||||
|
// Enabling client side object preparing for PUT operations.
|
||||||
|
cfgClientCut = "frostfs.client_cut"
|
||||||
|
|
||||||
// List of allowed AccessKeyID prefixes.
|
// List of allowed AccessKeyID prefixes.
|
||||||
cfgAllowedAccessKeyIDPrefixes = "allowed_access_key_id_prefixes"
|
cfgAllowedAccessKeyIDPrefixes = "allowed_access_key_id_prefixes"
|
||||||
|
|
|
@ -125,6 +125,8 @@ S3_GW_CORS_DEFAULT_MAX_AGE=600
|
||||||
# to consider PUT to FrostFS successful.
|
# to consider PUT to FrostFS successful.
|
||||||
# `0` or empty list means that object will be processed according to the container's placement policy
|
# `0` or empty list means that object will be processed according to the container's placement policy
|
||||||
S3_GW_FROSTFS_SET_COPIES_NUMBER=0
|
S3_GW_FROSTFS_SET_COPIES_NUMBER=0
|
||||||
|
# This flag enables client side object preparing.
|
||||||
|
S3_GW_FROSTFS_CLIENT_CUT=false
|
||||||
|
|
||||||
# List of allowed AccessKeyID prefixes
|
# List of allowed AccessKeyID prefixes
|
||||||
# If not set, S3 GW will accept all AccessKeyIDs
|
# If not set, S3 GW will accept all AccessKeyIDs
|
||||||
|
|
|
@ -150,6 +150,8 @@ frostfs:
|
||||||
# Numbers of the object copies (for each replica) to consider PUT to FrostFS successful.
|
# Numbers of the object copies (for each replica) to consider PUT to FrostFS successful.
|
||||||
# `[0]` or empty list means that object will be processed according to the container's placement policy
|
# `[0]` or empty list means that object will be processed according to the container's placement policy
|
||||||
set_copies_number: [0]
|
set_copies_number: [0]
|
||||||
|
# This flag enables client side object preparing.
|
||||||
|
client_cut: false
|
||||||
|
|
||||||
# List of allowed AccessKeyID prefixes
|
# List of allowed AccessKeyID prefixes
|
||||||
# If the parameter is omitted, S3 GW will accept all AccessKeyIDs
|
# If the parameter is omitted, S3 GW will accept all AccessKeyIDs
|
||||||
|
|
|
@ -500,17 +500,20 @@ tracing:
|
||||||
# `frostfs` section
|
# `frostfs` section
|
||||||
|
|
||||||
Contains parameters of requests to FrostFS.
|
Contains parameters of requests to FrostFS.
|
||||||
This value can be overridden with `X-Amz-Meta-Frostfs-Copies-Number` (value is comma separated numbers: `1,2,3`)
|
|
||||||
|
The `set_copies_number` value can be overridden with `X-Amz-Meta-Frostfs-Copies-Number` (value is comma separated numbers: `1,2,3`)
|
||||||
header for `PutObject`, `CopyObject`, `CreateMultipartUpload`.
|
header for `PutObject`, `CopyObject`, `CreateMultipartUpload`.
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
frostfs:
|
frostfs:
|
||||||
set_copies_number: [0]
|
set_copies_number: [0]
|
||||||
|
client_cut: false
|
||||||
```
|
```
|
||||||
|
|
||||||
| Parameter | Type | Default value | Description |
|
| Parameter | Type | SIGHUP reload | Default value | Description |
|
||||||
|---------------------|------------|---------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
|---------------------|------------|---------------|---------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
||||||
| `set_copies_number` | `[]uint32` | `[0]` | Numbers of the object copies (for each replica) to consider PUT to FrostFS successful. <br/>Default value `[0]` or empty list means that object will be processed according to the container's placement policy |
|
| `set_copies_number` | `[]uint32` | yes | `[0]` | Numbers of the object copies (for each replica) to consider PUT to FrostFS successful. <br/>Default value `[0]` or empty list means that object will be processed according to the container's placement policy |
|
||||||
|
| `client_cut` | `bool` | yes | `false` | This flag enables client side object preparing. |
|
||||||
|
|
||||||
# `resolve_bucket` section
|
# `resolve_bucket` section
|
||||||
|
|
||||||
|
|
2
go.sum
2
go.sum
|
@ -44,8 +44,6 @@ git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSV
|
||||||
git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU=
|
git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU=
|
||||||
git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 h1:aGQ6QaAnTerQ5Dq5b2/f9DUQtSqPkZZ/bkMx/HKuLCo=
|
git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 h1:aGQ6QaAnTerQ5Dq5b2/f9DUQtSqPkZZ/bkMx/HKuLCo=
|
||||||
git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6/go.mod h1:W8Nn08/l6aQ7UlIbpF7FsQou7TVpcRD1ZT1KG4TrFhE=
|
git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6/go.mod h1:W8Nn08/l6aQ7UlIbpF7FsQou7TVpcRD1ZT1KG4TrFhE=
|
||||||
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230821073319-342524159ac3 h1:GBRTOTRrtIvxi2TgxG7z/J7uRXiyb1SxR4247FaYCgU=
|
|
||||||
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230821073319-342524159ac3/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw=
|
|
||||||
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230821090303-202412230a05 h1:OuViMF54N87FXmaBEpYw3jhzaLrJ/EWOlPL1wUkimE0=
|
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230821090303-202412230a05 h1:OuViMF54N87FXmaBEpYw3jhzaLrJ/EWOlPL1wUkimE0=
|
||||||
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230821090303-202412230a05/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw=
|
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230821090303-202412230a05/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw=
|
||||||
git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc=
|
git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc=
|
||||||
|
|
|
@ -243,6 +243,7 @@ func (x *FrostFS) CreateObject(ctx context.Context, prm layer.PrmObjectCreate) (
|
||||||
prmPut.SetHeader(*obj)
|
prmPut.SetHeader(*obj)
|
||||||
prmPut.SetPayload(prm.Payload)
|
prmPut.SetPayload(prm.Payload)
|
||||||
prmPut.SetCopiesNumberVector(prm.CopiesNumber)
|
prmPut.SetCopiesNumberVector(prm.CopiesNumber)
|
||||||
|
prmPut.SetClientCut(prm.ClientCut)
|
||||||
|
|
||||||
if prm.BearerToken != nil {
|
if prm.BearerToken != nil {
|
||||||
prmPut.UseBearer(*prm.BearerToken)
|
prmPut.UseBearer(*prm.BearerToken)
|
||||||
|
|
Loading…
Reference in a new issue