forked from TrueCloudLab/frostfs-s3-gw
[#192] Support client cut
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
This commit is contained in:
parent
fcf1c45ad2
commit
6b1f365e65
11 changed files with 38 additions and 6 deletions
|
@ -26,6 +26,7 @@ This document outlines major changes between releases.
|
|||
- Support multiple version credentials using GSet (#135)
|
||||
- Implement chunk uploading (#106)
|
||||
- Add new `kludge.bypass_content_encoding_check_in_chunks` config param (#146)
|
||||
- Add new `frostfs.client_cut` config param (#192)
|
||||
|
||||
### Changed
|
||||
- Update prometheus to v1.15.0 (#94)
|
||||
|
|
|
@ -111,6 +111,9 @@ type PrmObjectCreate struct {
|
|||
|
||||
// Number of object copies that is enough to consider put successful.
|
||||
CopiesNumber []uint32
|
||||
|
||||
// Enables client side object preparing.
|
||||
ClientCut bool
|
||||
}
|
||||
|
||||
// PrmObjectDelete groups parameters of FrostFS.DeleteObject operation.
|
||||
|
|
|
@ -45,6 +45,10 @@ type (
|
|||
Resolve(ctx context.Context, name string) (cid.ID, error)
|
||||
}
|
||||
|
||||
FeatureSettings interface {
|
||||
ClientCut() bool
|
||||
}
|
||||
|
||||
layer struct {
|
||||
frostFS FrostFS
|
||||
gateOwner user.ID
|
||||
|
@ -54,6 +58,7 @@ type (
|
|||
ncontroller EventListener
|
||||
cache *Cache
|
||||
treeService TreeService
|
||||
features FeatureSettings
|
||||
}
|
||||
|
||||
Config struct {
|
||||
|
@ -63,6 +68,7 @@ type (
|
|||
AnonKey AnonymousKey
|
||||
Resolver BucketResolver
|
||||
TreeService TreeService
|
||||
Features FeatureSettings
|
||||
}
|
||||
|
||||
// AnonymousKey contains data for anonymous requests.
|
||||
|
@ -301,6 +307,7 @@ func NewLayer(log *zap.Logger, frostFS FrostFS, config *Config) Client {
|
|||
resolver: config.Resolver,
|
||||
cache: NewCache(config.Caches),
|
||||
treeService: config.TreeService,
|
||||
features: config.Features,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -458,6 +458,7 @@ func (n *layer) objectDelete(ctx context.Context, bktInfo *data.BucketInfo, idOb
|
|||
// Returns object ID and payload sha256 hash.
|
||||
func (n *layer) objectPutAndHash(ctx context.Context, prm PrmObjectCreate, bktInfo *data.BucketInfo) (uint64, oid.ID, []byte, error) {
|
||||
n.prepareAuthParameters(ctx, &prm.PrmAuth, bktInfo.Owner)
|
||||
prm.ClientCut = n.features.ClientCut()
|
||||
var size uint64
|
||||
hash := sha256.New()
|
||||
prm.Payload = wrapReader(prm.Payload, 64*1024, func(buf []byte) {
|
||||
|
|
|
@ -71,6 +71,7 @@ type (
|
|||
xmlDecoder *xml.DecoderProvider
|
||||
maxClient maxClientsConfig
|
||||
bypassContentEncodingInChunks atomic.Bool
|
||||
clientCut atomic.Bool
|
||||
}
|
||||
|
||||
maxClientsConfig struct {
|
||||
|
@ -144,6 +145,7 @@ func (a *App) initLayer(ctx context.Context) {
|
|||
GateOwner: gateOwner,
|
||||
Resolver: a.bucketResolver,
|
||||
TreeService: tree.NewTree(services.NewPoolWrapper(a.treePool), a.log),
|
||||
Features: a.settings,
|
||||
}
|
||||
|
||||
// prepare object layer
|
||||
|
@ -176,6 +178,7 @@ func newAppSettings(log *Logger, v *viper.Viper) *appSettings {
|
|||
}
|
||||
|
||||
settings.setBypassContentEncodingInChunks(v.GetBool(cfgKludgeBypassContentEncodingCheckInChunks))
|
||||
settings.setClientCut(v.GetBool(cfgClientCut))
|
||||
|
||||
return settings
|
||||
}
|
||||
|
@ -188,6 +191,14 @@ func (s *appSettings) setBypassContentEncodingInChunks(bypass bool) {
|
|||
s.bypassContentEncodingInChunks.Store(bypass)
|
||||
}
|
||||
|
||||
func (s *appSettings) ClientCut() bool {
|
||||
return s.clientCut.Load()
|
||||
}
|
||||
|
||||
func (s *appSettings) setClientCut(clientCut bool) {
|
||||
s.clientCut.Store(clientCut)
|
||||
}
|
||||
|
||||
func (a *App) initAPI(ctx context.Context) {
|
||||
a.initLayer(ctx)
|
||||
a.initHandler()
|
||||
|
@ -568,6 +579,7 @@ func (a *App) updateSettings() {
|
|||
|
||||
a.settings.xmlDecoder.UseDefaultNamespaceForCompleteMultipart(a.cfg.GetBool(cfgKludgeUseDefaultXMLNSForCompleteMultipartUpload))
|
||||
a.settings.setBypassContentEncodingInChunks(a.cfg.GetBool(cfgKludgeBypassContentEncodingCheckInChunks))
|
||||
a.settings.setClientCut(a.cfg.GetBool(cfgClientCut))
|
||||
}
|
||||
|
||||
func (a *App) startServices() {
|
||||
|
|
|
@ -138,6 +138,8 @@ const ( // Settings.
|
|||
// Configuration of parameters of requests to FrostFS.
|
||||
// Number of the object copies to consider PUT to FrostFS successful.
|
||||
cfgSetCopiesNumber = "frostfs.set_copies_number"
|
||||
// Enabling client side object preparing for PUT operations.
|
||||
cfgClientCut = "frostfs.client_cut"
|
||||
|
||||
// List of allowed AccessKeyID prefixes.
|
||||
cfgAllowedAccessKeyIDPrefixes = "allowed_access_key_id_prefixes"
|
||||
|
|
|
@ -125,6 +125,8 @@ S3_GW_CORS_DEFAULT_MAX_AGE=600
|
|||
# to consider PUT to FrostFS successful.
|
||||
# `0` or empty list means that object will be processed according to the container's placement policy
|
||||
S3_GW_FROSTFS_SET_COPIES_NUMBER=0
|
||||
# This flag enables client side object preparing.
|
||||
S3_GW_FROSTFS_CLIENT_CUT=false
|
||||
|
||||
# List of allowed AccessKeyID prefixes
|
||||
# If not set, S3 GW will accept all AccessKeyIDs
|
||||
|
|
|
@ -150,6 +150,8 @@ frostfs:
|
|||
# Numbers of the object copies (for each replica) to consider PUT to FrostFS successful.
|
||||
# `[0]` or empty list means that object will be processed according to the container's placement policy
|
||||
set_copies_number: [0]
|
||||
# This flag enables client side object preparing.
|
||||
client_cut: false
|
||||
|
||||
# List of allowed AccessKeyID prefixes
|
||||
# If the parameter is omitted, S3 GW will accept all AccessKeyIDs
|
||||
|
|
|
@ -500,17 +500,20 @@ tracing:
|
|||
# `frostfs` section
|
||||
|
||||
Contains parameters of requests to FrostFS.
|
||||
This value can be overridden with `X-Amz-Meta-Frostfs-Copies-Number` (value is comma separated numbers: `1,2,3`)
|
||||
|
||||
The `set_copies_number` value can be overridden with `X-Amz-Meta-Frostfs-Copies-Number` (value is comma separated numbers: `1,2,3`)
|
||||
header for `PutObject`, `CopyObject`, `CreateMultipartUpload`.
|
||||
|
||||
```yaml
|
||||
frostfs:
|
||||
set_copies_number: [0]
|
||||
client_cut: false
|
||||
```
|
||||
|
||||
| Parameter | Type | Default value | Description |
|
||||
|---------------------|------------|---------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| `set_copies_number` | `[]uint32` | `[0]` | Numbers of the object copies (for each replica) to consider PUT to FrostFS successful. <br/>Default value `[0]` or empty list means that object will be processed according to the container's placement policy |
|
||||
| Parameter | Type | SIGHUP reload | Default value | Description |
|
||||
|---------------------|------------|---------------|---------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| `set_copies_number` | `[]uint32` | yes | `[0]` | Numbers of the object copies (for each replica) to consider PUT to FrostFS successful. <br/>Default value `[0]` or empty list means that object will be processed according to the container's placement policy |
|
||||
| `client_cut` | `bool` | yes | `false` | This flag enables client side object preparing. |
|
||||
|
||||
# `resolve_bucket` section
|
||||
|
||||
|
|
2
go.sum
2
go.sum
|
@ -44,8 +44,6 @@ git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSV
|
|||
git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU=
|
||||
git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 h1:aGQ6QaAnTerQ5Dq5b2/f9DUQtSqPkZZ/bkMx/HKuLCo=
|
||||
git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6/go.mod h1:W8Nn08/l6aQ7UlIbpF7FsQou7TVpcRD1ZT1KG4TrFhE=
|
||||
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230821073319-342524159ac3 h1:GBRTOTRrtIvxi2TgxG7z/J7uRXiyb1SxR4247FaYCgU=
|
||||
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230821073319-342524159ac3/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw=
|
||||
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230821090303-202412230a05 h1:OuViMF54N87FXmaBEpYw3jhzaLrJ/EWOlPL1wUkimE0=
|
||||
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230821090303-202412230a05/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw=
|
||||
git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc=
|
||||
|
|
|
@ -243,6 +243,7 @@ func (x *FrostFS) CreateObject(ctx context.Context, prm layer.PrmObjectCreate) (
|
|||
prmPut.SetHeader(*obj)
|
||||
prmPut.SetPayload(prm.Payload)
|
||||
prmPut.SetCopiesNumberVector(prm.CopiesNumber)
|
||||
prmPut.SetClientCut(prm.ClientCut)
|
||||
|
||||
if prm.BearerToken != nil {
|
||||
prmPut.UseBearer(*prm.BearerToken)
|
||||
|
|
Loading…
Reference in a new issue