From 510a20fd668a121aac147e7b7421914e87dac9f7 Mon Sep 17 00:00:00 2001 From: Pavel Pogodaev Date: Wed, 27 Nov 2024 11:22:46 +0300 Subject: [PATCH] [#147] Add Kludge profiles Signed-off-by: Pavel Pogodaev --- api/handler/api.go | 9 ++- api/handler/cors.go | 1 + api/handler/delete.go | 2 +- api/handler/handlers_test.go | 4 +- api/handler/lifecycle.go | 2 +- api/handler/locking.go | 6 +- api/handler/multipart_upload.go | 2 +- api/handler/put.go | 8 ++- api/handler/versioning.go | 2 +- api/layer/cors.go | 2 +- api/layer/layer.go | 3 +- api/middleware/policy.go | 4 +- api/router_mock_test.go | 112 ++++++++++++++++---------------- cmd/s3-gw/app.go | 37 +++++++++-- cmd/s3-gw/app_settings.go | 33 ++++++++++ cmd/s3-gw/decoder_test.go | 2 +- config/config.env | 4 ++ config/config.yaml | 7 ++ docs/configuration.md | 9 ++- 19 files changed, 168 insertions(+), 81 deletions(-) diff --git a/api/handler/api.go b/api/handler/api.go index 559977a..100bae8 100644 --- a/api/handler/api.go +++ b/api/handler/api.go @@ -32,11 +32,11 @@ type ( PlacementPolicy(namespace, constraint string) (netmap.PlacementPolicy, bool) CopiesNumbers(namespace, constraint string) ([]uint32, bool) DefaultCopiesNumbers(namespace string) []uint32 - NewXMLDecoder(io.Reader) *xml.Decoder + NewXMLDecoder(reader io.Reader, agent string) *xml.Decoder DefaultMaxAge() int ResolveZoneList() []string IsResolveListAllow() bool - BypassContentEncodingInChunks() bool + BypassContentEncodingInChunks(agent string) bool MD5Enabled() bool RetryMaxAttempts() int RetryMaxBackoff() time.Duration @@ -59,6 +59,11 @@ type ( type RetryStrategy string +type KludgeParams struct { + IsSet bool + Value bool +} + const ( RetryStrategyExponential = "exponential" RetryStrategyConstant = "constant" diff --git a/api/handler/cors.go b/api/handler/cors.go index ddbc829..6a671a5 100644 --- a/api/handler/cors.go +++ b/api/handler/cors.go @@ -55,6 +55,7 @@ func (h *handler) PutBucketCorsHandler(w http.ResponseWriter, r *http.Request) { BktInfo: bktInfo, Reader: r.Body, NewDecoder: h.cfg.NewXMLDecoder, + UserAgent: r.UserAgent(), } p.CopiesNumbers, err = h.pickCopiesNumbers(parseMetadata(r), reqInfo.Namespace, bktInfo.LocationConstraint) diff --git a/api/handler/delete.go b/api/handler/delete.go index 8ac9fa9..67b025d 100644 --- a/api/handler/delete.go +++ b/api/handler/delete.go @@ -147,7 +147,7 @@ func (h *handler) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Re // Unmarshal list of keys to be deleted. requested := &DeleteObjectsRequest{} - if err := h.cfg.NewXMLDecoder(r.Body).Decode(requested); err != nil { + if err := h.cfg.NewXMLDecoder(r.Body, r.UserAgent()).Decode(requested); err != nil { h.logAndSendError(ctx, w, "couldn't decode body", reqInfo, fmt.Errorf("%w: %s", errors.GetAPIError(errors.ErrMalformedXML), err.Error())) return } diff --git a/api/handler/handlers_test.go b/api/handler/handlers_test.go index e5a224b..57f9741 100644 --- a/api/handler/handlers_test.go +++ b/api/handler/handlers_test.go @@ -97,11 +97,11 @@ func (c *configMock) DefaultCopiesNumbers(_ string) []uint32 { return c.defaultCopiesNumbers } -func (c *configMock) NewXMLDecoder(r io.Reader) *xml.Decoder { +func (c *configMock) NewXMLDecoder(r io.Reader, _ string) *xml.Decoder { return xml.NewDecoder(r) } -func (c *configMock) BypassContentEncodingInChunks() bool { +func (c *configMock) BypassContentEncodingInChunks(_ string) bool { return c.bypassContentEncodingInChunks } diff --git a/api/handler/lifecycle.go b/api/handler/lifecycle.go index 3aa3ccc..4ebdb2e 100644 --- a/api/handler/lifecycle.go +++ b/api/handler/lifecycle.go @@ -69,7 +69,7 @@ func (h *handler) PutBucketLifecycleHandler(w http.ResponseWriter, r *http.Reque } cfg := new(data.LifecycleConfiguration) - if err = h.cfg.NewXMLDecoder(tee).Decode(cfg); err != nil { + if err = h.cfg.NewXMLDecoder(tee, r.UserAgent()).Decode(cfg); err != nil { h.logAndSendError(ctx, w, "could not decode body", reqInfo, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrMalformedXML), err.Error())) return } diff --git a/api/handler/locking.go b/api/handler/locking.go index bc5813d..e9a0a23 100644 --- a/api/handler/locking.go +++ b/api/handler/locking.go @@ -42,7 +42,7 @@ func (h *handler) PutBucketObjectLockConfigHandler(w http.ResponseWriter, r *htt } lockingConf := &data.ObjectLockConfiguration{} - if err = h.cfg.NewXMLDecoder(r.Body).Decode(lockingConf); err != nil { + if err = h.cfg.NewXMLDecoder(r.Body, r.UserAgent()).Decode(lockingConf); err != nil { h.logAndSendError(ctx, w, "couldn't parse locking configuration", reqInfo, err) return } @@ -124,7 +124,7 @@ func (h *handler) PutObjectLegalHoldHandler(w http.ResponseWriter, r *http.Reque } legalHold := &data.LegalHold{} - if err = h.cfg.NewXMLDecoder(r.Body).Decode(legalHold); err != nil { + if err = h.cfg.NewXMLDecoder(r.Body, r.UserAgent()).Decode(legalHold); err != nil { h.logAndSendError(ctx, w, "couldn't parse legal hold configuration", reqInfo, err) return } @@ -214,7 +214,7 @@ func (h *handler) PutObjectRetentionHandler(w http.ResponseWriter, r *http.Reque } retention := &data.Retention{} - if err = h.cfg.NewXMLDecoder(r.Body).Decode(retention); err != nil { + if err = h.cfg.NewXMLDecoder(r.Body, r.UserAgent()).Decode(retention); err != nil { h.logAndSendError(ctx, w, "couldn't parse object retention", reqInfo, err) return } diff --git a/api/handler/multipart_upload.go b/api/handler/multipart_upload.go index bb10927..725052d 100644 --- a/api/handler/multipart_upload.go +++ b/api/handler/multipart_upload.go @@ -401,7 +401,7 @@ func (h *handler) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http. ) reqBody := new(CompleteMultipartUpload) - if err = h.cfg.NewXMLDecoder(r.Body).Decode(reqBody); err != nil { + if err = h.cfg.NewXMLDecoder(r.Body, r.UserAgent()).Decode(reqBody); err != nil { h.logAndSendError(ctx, w, "could not read complete multipart upload xml", reqInfo, fmt.Errorf("%w: %s", errors.GetAPIError(errors.ErrMalformedXML), err.Error()), additional...) return diff --git a/api/handler/put.go b/api/handler/put.go index d08389f..58518f6 100644 --- a/api/handler/put.go +++ b/api/handler/put.go @@ -331,7 +331,9 @@ func (h *handler) getBodyReader(r *http.Request) (io.ReadCloser, error) { } r.Header.Set(api.ContentEncoding, strings.Join(resultContentEncoding, ",")) - if !chunkedEncoding && !h.cfg.BypassContentEncodingInChunks() { + defBypass := h.cfg.BypassContentEncodingInChunks(r.UserAgent()) + + if !chunkedEncoding && !defBypass { return nil, fmt.Errorf("%w: request is not chunk encoded, encodings '%s'", apierr.GetAPIError(apierr.ErrInvalidEncodingMethod), strings.Join(encodings, ",")) } @@ -442,7 +444,7 @@ func (h *handler) PostObject(w http.ResponseWriter, r *http.Request) { if tagging := auth.MultipartFormValue(r, "tagging"); tagging != "" { buffer := bytes.NewBufferString(tagging) tags := new(data.Tagging) - if err = h.cfg.NewXMLDecoder(buffer).Decode(tags); err != nil { + if err = h.cfg.NewXMLDecoder(buffer, r.UserAgent()).Decode(tags); err != nil { h.logAndSendError(ctx, w, "could not decode tag set", reqInfo, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrMalformedXML), err.Error())) return @@ -1033,7 +1035,7 @@ func (h *handler) parseLocationConstraint(r *http.Request) (*createBucketParams, } params := new(createBucketParams) - if err := h.cfg.NewXMLDecoder(r.Body).Decode(params); err != nil { + if err := h.cfg.NewXMLDecoder(r.Body, r.UserAgent()).Decode(params); err != nil { return nil, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrMalformedXML), err.Error()) } return params, nil diff --git a/api/handler/versioning.go b/api/handler/versioning.go index d8b5bda..bf9c895 100644 --- a/api/handler/versioning.go +++ b/api/handler/versioning.go @@ -14,7 +14,7 @@ func (h *handler) PutBucketVersioningHandler(w http.ResponseWriter, r *http.Requ reqInfo := middleware.GetReqInfo(ctx) configuration := new(VersioningConfiguration) - if err := h.cfg.NewXMLDecoder(r.Body).Decode(configuration); err != nil { + if err := h.cfg.NewXMLDecoder(r.Body, r.UserAgent()).Decode(configuration); err != nil { h.logAndSendError(ctx, w, "couldn't decode versioning configuration", reqInfo, errors.GetAPIError(errors.ErrIllegalVersioningConfigurationException)) return } diff --git a/api/layer/cors.go b/api/layer/cors.go index 49b6cc4..ebf0a19 100644 --- a/api/layer/cors.go +++ b/api/layer/cors.go @@ -28,7 +28,7 @@ func (n *Layer) PutBucketCORS(ctx context.Context, p *PutCORSParams) error { cors = &data.CORSConfiguration{} ) - if err := p.NewDecoder(tee).Decode(cors); err != nil { + if err := p.NewDecoder(tee, p.UserAgent).Decode(cors); err != nil { return fmt.Errorf("xml decode cors: %w", err) } diff --git a/api/layer/layer.go b/api/layer/layer.go index cafbb66..1589c7a 100644 --- a/api/layer/layer.go +++ b/api/layer/layer.go @@ -149,7 +149,8 @@ type ( BktInfo *data.BucketInfo Reader io.Reader CopiesNumbers []uint32 - NewDecoder func(io.Reader) *xml.Decoder + NewDecoder func(io.Reader, string) *xml.Decoder + UserAgent string } // CopyObjectParams stores object copy request parameters. diff --git a/api/middleware/policy.go b/api/middleware/policy.go index 9e36f0b..9faf069 100644 --- a/api/middleware/policy.go +++ b/api/middleware/policy.go @@ -61,7 +61,7 @@ type FrostFSIDInformer interface { } type XMLDecoder interface { - NewXMLDecoder(io.Reader) *xml.Decoder + NewXMLDecoder(io.Reader, string) *xml.Decoder } type ResourceTagging interface { @@ -476,7 +476,7 @@ func determineRequestTags(r *http.Request, decoder XMLDecoder, op string) (map[s if strings.HasSuffix(op, PutObjectTaggingOperation) || strings.HasSuffix(op, PutBucketTaggingOperation) { tagging := new(data.Tagging) - if err := decoder.NewXMLDecoder(r.Body).Decode(tagging); err != nil { + if err := decoder.NewXMLDecoder(r.Body, r.UserAgent()).Decode(tagging); err != nil { return nil, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrMalformedXML), err.Error()) } GetReqInfo(r.Context()).Tagging = tagging diff --git a/api/router_mock_test.go b/api/router_mock_test.go index 4f237f9..43ea707 100644 --- a/api/router_mock_test.go +++ b/api/router_mock_test.go @@ -151,7 +151,7 @@ func (f *frostFSIDMock) GetUserGroupIDsAndClaims(util.Uint160) ([]string, map[st type xmlMock struct { } -func (m *xmlMock) NewXMLDecoder(r io.Reader) *xml.Decoder { +func (m *xmlMock) NewXMLDecoder(r io.Reader, _ string) *xml.Decoder { return xml.NewDecoder(r) } @@ -187,47 +187,47 @@ type handlerResult struct { } func (h *handlerMock) HeadObjectHandler(http.ResponseWriter, *http.Request) { - //TODO implement me + // TODO implement me panic("implement me") } func (h *handlerMock) GetObjectACLHandler(http.ResponseWriter, *http.Request) { - //TODO implement me + // TODO implement me panic("implement me") } func (h *handlerMock) PutObjectACLHandler(http.ResponseWriter, *http.Request) { - //TODO implement me + // TODO implement me panic("implement me") } func (h *handlerMock) GetObjectTaggingHandler(http.ResponseWriter, *http.Request) { - //TODO implement me + // TODO implement me panic("implement me") } func (h *handlerMock) PutObjectTaggingHandler(http.ResponseWriter, *http.Request) { - //TODO implement me + // TODO implement me panic("implement me") } func (h *handlerMock) DeleteObjectTaggingHandler(http.ResponseWriter, *http.Request) { - //TODO implement me + // TODO implement me panic("implement me") } func (h *handlerMock) SelectObjectContentHandler(http.ResponseWriter, *http.Request) { - //TODO implement me + // TODO implement me panic("implement me") } func (h *handlerMock) GetObjectRetentionHandler(http.ResponseWriter, *http.Request) { - //TODO implement me + // TODO implement me panic("implement me") } func (h *handlerMock) GetObjectLegalHoldHandler(http.ResponseWriter, *http.Request) { - //TODO implement me + // TODO implement me panic("implement me") } @@ -241,22 +241,22 @@ func (h *handlerMock) GetObjectHandler(w http.ResponseWriter, r *http.Request) { } func (h *handlerMock) GetObjectAttributesHandler(http.ResponseWriter, *http.Request) { - //TODO implement me + // TODO implement me panic("implement me") } func (h *handlerMock) CopyObjectHandler(http.ResponseWriter, *http.Request) { - //TODO implement me + // TODO implement me panic("implement me") } func (h *handlerMock) PutObjectRetentionHandler(http.ResponseWriter, *http.Request) { - //TODO implement me + // TODO implement me panic("implement me") } func (h *handlerMock) PutObjectLegalHoldHandler(http.ResponseWriter, *http.Request) { - //TODO implement me + // TODO implement me panic("implement me") } @@ -279,117 +279,117 @@ func (h *handlerMock) DeleteObjectHandler(w http.ResponseWriter, r *http.Request } func (h *handlerMock) GetBucketLocationHandler(http.ResponseWriter, *http.Request) { - //TODO implement me + // TODO implement me panic("implement me") } func (h *handlerMock) GetBucketPolicyStatusHandler(http.ResponseWriter, *http.Request) { - //TODO implement me + // TODO implement me panic("implement me") } func (h *handlerMock) GetBucketPolicyHandler(http.ResponseWriter, *http.Request) { - //TODO implement me + // TODO implement me panic("implement me") } func (h *handlerMock) GetBucketLifecycleHandler(http.ResponseWriter, *http.Request) { - //TODO implement me + // TODO implement me panic("implement me") } func (h *handlerMock) GetBucketEncryptionHandler(http.ResponseWriter, *http.Request) { - //TODO implement me + // TODO implement me panic("implement me") } func (h *handlerMock) GetBucketACLHandler(http.ResponseWriter, *http.Request) { - //TODO implement me + // TODO implement me panic("implement me") } func (h *handlerMock) PutBucketACLHandler(http.ResponseWriter, *http.Request) { - //TODO implement me + // TODO implement me panic("implement me") } func (h *handlerMock) GetBucketCorsHandler(http.ResponseWriter, *http.Request) { - //TODO implement me + // TODO implement me panic("implement me") } func (h *handlerMock) PutBucketCorsHandler(http.ResponseWriter, *http.Request) { - //TODO implement me + // TODO implement me panic("implement me") } func (h *handlerMock) DeleteBucketCorsHandler(http.ResponseWriter, *http.Request) { - //TODO implement me + // TODO implement me panic("implement me") } func (h *handlerMock) GetBucketWebsiteHandler(http.ResponseWriter, *http.Request) { - //TODO implement me + // TODO implement me panic("implement me") } func (h *handlerMock) GetBucketAccelerateHandler(http.ResponseWriter, *http.Request) { - //TODO implement me + // TODO implement me panic("implement me") } func (h *handlerMock) GetBucketRequestPaymentHandler(http.ResponseWriter, *http.Request) { - //TODO implement me + // TODO implement me panic("implement me") } func (h *handlerMock) GetBucketLoggingHandler(http.ResponseWriter, *http.Request) { - //TODO implement me + // TODO implement me panic("implement me") } func (h *handlerMock) GetBucketReplicationHandler(http.ResponseWriter, *http.Request) { - //TODO implement me + // TODO implement me panic("implement me") } func (h *handlerMock) GetBucketTaggingHandler(http.ResponseWriter, *http.Request) { - //TODO implement me + // TODO implement me panic("implement me") } func (h *handlerMock) DeleteBucketWebsiteHandler(http.ResponseWriter, *http.Request) { - //TODO implement me + // TODO implement me panic("implement me") } func (h *handlerMock) DeleteBucketTaggingHandler(http.ResponseWriter, *http.Request) { - //TODO implement me + // TODO implement me panic("implement me") } func (h *handlerMock) GetBucketObjectLockConfigHandler(http.ResponseWriter, *http.Request) { - //TODO implement me + // TODO implement me panic("implement me") } func (h *handlerMock) GetBucketVersioningHandler(http.ResponseWriter, *http.Request) { - //TODO implement me + // TODO implement me panic("implement me") } func (h *handlerMock) GetBucketNotificationHandler(http.ResponseWriter, *http.Request) { - //TODO implement me + // TODO implement me panic("implement me") } func (h *handlerMock) ListenBucketNotificationHandler(http.ResponseWriter, *http.Request) { - //TODO implement me + // TODO implement me panic("implement me") } func (h *handlerMock) ListObjectsV2MHandler(http.ResponseWriter, *http.Request) { - //TODO implement me + // TODO implement me panic("implement me") } @@ -403,7 +403,7 @@ func (h *handlerMock) ListObjectsV2Handler(w http.ResponseWriter, r *http.Reques } func (h *handlerMock) ListBucketObjectVersionsHandler(http.ResponseWriter, *http.Request) { - //TODO implement me + // TODO implement me panic("implement me") } @@ -417,22 +417,22 @@ func (h *handlerMock) ListObjectsV1Handler(w http.ResponseWriter, r *http.Reques } func (h *handlerMock) PutBucketLifecycleHandler(http.ResponseWriter, *http.Request) { - //TODO implement me + // TODO implement me panic("implement me") } func (h *handlerMock) PutBucketEncryptionHandler(http.ResponseWriter, *http.Request) { - //TODO implement me + // TODO implement me panic("implement me") } func (h *handlerMock) PutBucketPolicyHandler(http.ResponseWriter, *http.Request) { - //TODO implement me + // TODO implement me panic("implement me") } func (h *handlerMock) PutBucketObjectLockConfigHandler(http.ResponseWriter, *http.Request) { - //TODO implement me + // TODO implement me panic("implement me") } @@ -446,12 +446,12 @@ func (h *handlerMock) PutBucketTaggingHandler(w http.ResponseWriter, r *http.Req } func (h *handlerMock) PutBucketVersioningHandler(http.ResponseWriter, *http.Request) { - //TODO implement me + // TODO implement me panic("implement me") } func (h *handlerMock) PutBucketNotificationHandler(http.ResponseWriter, *http.Request) { - //TODO implement me + // TODO implement me panic("implement me") } @@ -480,32 +480,32 @@ func (h *handlerMock) HeadBucketHandler(w http.ResponseWriter, r *http.Request) } func (h *handlerMock) PostObject(http.ResponseWriter, *http.Request) { - //TODO implement me + // TODO implement me panic("implement me") } func (h *handlerMock) DeleteMultipleObjectsHandler(http.ResponseWriter, *http.Request) { - //TODO implement me + // TODO implement me panic("implement me") } func (h *handlerMock) DeleteBucketPolicyHandler(http.ResponseWriter, *http.Request) { - //TODO implement me + // TODO implement me panic("implement me") } func (h *handlerMock) DeleteBucketLifecycleHandler(http.ResponseWriter, *http.Request) { - //TODO implement me + // TODO implement me panic("implement me") } func (h *handlerMock) DeleteBucketEncryptionHandler(http.ResponseWriter, *http.Request) { - //TODO implement me + // TODO implement me panic("implement me") } func (h *handlerMock) DeleteBucketHandler(http.ResponseWriter, *http.Request) { - //TODO implement me + // TODO implement me panic("implement me") } @@ -519,7 +519,7 @@ func (h *handlerMock) ListBucketsHandler(w http.ResponseWriter, r *http.Request) } func (h *handlerMock) Preflight(http.ResponseWriter, *http.Request) { - //TODO implement me + // TODO implement me panic("implement me") } @@ -527,7 +527,7 @@ func (h *handlerMock) AppendCORSHeaders(http.ResponseWriter, *http.Request) { } func (h *handlerMock) CreateMultipartUploadHandler(http.ResponseWriter, *http.Request) { - //TODO implement me + // TODO implement me panic("implement me") } @@ -541,22 +541,22 @@ func (h *handlerMock) UploadPartHandler(w http.ResponseWriter, r *http.Request) } func (h *handlerMock) UploadPartCopy(http.ResponseWriter, *http.Request) { - //TODO implement me + // TODO implement me panic("implement me") } func (h *handlerMock) CompleteMultipartUploadHandler(http.ResponseWriter, *http.Request) { - //TODO implement me + // TODO implement me panic("implement me") } func (h *handlerMock) AbortMultipartUploadHandler(http.ResponseWriter, *http.Request) { - //TODO implement me + // TODO implement me panic("implement me") } func (h *handlerMock) ListPartsHandler(http.ResponseWriter, *http.Request) { - //TODO implement me + // TODO implement me panic("implement me") } diff --git a/cmd/s3-gw/app.go b/cmd/s3-gw/app.go index dc5677b..31703b9 100644 --- a/cmd/s3-gw/app.go +++ b/cmd/s3-gw/app.go @@ -112,6 +112,7 @@ type ( namespaces Namespaces defaultXMLNS bool bypassContentEncodingInChunks bool + kludgeProfiles map[string]*KludgeParams clientCut bool maxBufferSizeForPut uint64 md5Enabled bool @@ -297,6 +298,7 @@ func (s *appSettings) update(v *viper.Viper, log *zap.Logger) { vhsNamespacesEnabled := s.prepareVHSNamespaces(v, log, defaultNamespaces) defaultXMLNS := v.GetBool(cfgKludgeUseDefaultXMLNS) bypassContentEncodingInChunks := v.GetBool(cfgKludgeBypassContentEncodingCheckInChunks) + kludgeProfiles := fetchKludgeProfiles(v) clientCut := v.GetBool(cfgClientCut) maxBufferSizeForPut := v.GetUint64(cfgBufferMaxSizeForPut) md5Enabled := v.GetBool(cfgMD5Enabled) @@ -332,6 +334,7 @@ func (s *appSettings) update(v *viper.Viper, log *zap.Logger) { s.namespaces = nsConfig.Namespaces s.defaultXMLNS = defaultXMLNS s.bypassContentEncodingInChunks = bypassContentEncodingInChunks + s.kludgeProfiles = kludgeProfiles s.clientCut = clientCut s.maxBufferSizeForPut = maxBufferSizeForPut s.md5Enabled = md5Enabled @@ -392,10 +395,25 @@ func (s *appSettings) VHSNamespacesEnabled() map[string]bool { return s.vhsNamespacesEnabled } -func (s *appSettings) BypassContentEncodingInChunks() bool { +func (s *appSettings) BypassContentEncodingInChunks(agent string) bool { s.mu.RLock() defer s.mu.RUnlock() - return s.bypassContentEncodingInChunks + + var profile *KludgeParams + profiles := s.kludgeProfiles + for p := range profiles { + if strings.Contains(agent, p) { + profile = profiles[p] + } + } + + return s.bypassContentEncodingInChunks || (profile != nil && profile.BypassContentEncodingCheckInChunks) +} + +func (s *appSettings) KludgeProfiles() map[string]*KludgeParams { + s.mu.RLock() + defer s.mu.RUnlock() + return s.kludgeProfiles } func (s *appSettings) ClientCut() bool { @@ -445,7 +463,7 @@ func (s *appSettings) LogHTTPConfig() s3middleware.LogHTTPConfig { return s.httpLogging } -func (s *appSettings) NewXMLDecoder(r io.Reader) *xml.Decoder { +func (s *appSettings) NewXMLDecoder(r io.Reader, agent string) *xml.Decoder { dec := xml.NewDecoder(r) dec.CharsetReader = func(charset string, reader io.Reader) (io.Reader, error) { enc, err := ianaindex.IANA.Encoding(charset) @@ -456,10 +474,19 @@ func (s *appSettings) NewXMLDecoder(r io.Reader) *xml.Decoder { } s.mu.RLock() - if s.defaultXMLNS { + defer s.mu.RUnlock() + + var profile *KludgeParams + for p := range s.kludgeProfiles { + if strings.Contains(agent, p) { + profile = s.kludgeProfiles[p] + break + } + } + + if s.defaultXMLNS || (profile != nil && profile.UseDefaultXMLNS) { dec.DefaultSpace = awsDefaultNamespace } - s.mu.RUnlock() return dec } diff --git a/cmd/s3-gw/app_settings.go b/cmd/s3-gw/app_settings.go index edf8769..06502b7 100644 --- a/cmd/s3-gw/app_settings.go +++ b/cmd/s3-gw/app_settings.go @@ -197,6 +197,7 @@ const ( // Settings. cfgKludgeUseDefaultXMLNS = "kludge.use_default_xmlns" cfgKludgeBypassContentEncodingCheckInChunks = "kludge.bypass_content_encoding_check_in_chunks" cfgKludgeDefaultNamespaces = "kludge.default_namespaces" + cfgKludgeProfile = "kludge.profile" // Web. cfgWebReadTimeout = "web.read_timeout" cfgWebReadHeaderTimeout = "web.read_header_timeout" @@ -563,6 +564,38 @@ func fetchDefaultCopiesNumbers(l *zap.Logger, v *viper.Viper) []uint32 { return result } +type KludgeParams struct { + UseDefaultXMLNS bool + BypassContentEncodingCheckInChunks bool +} + +func fetchKludgeProfiles(v *viper.Viper) map[string]*KludgeParams { + kludgeProfiles := make(map[string]*KludgeParams) + for i := 0; ; i++ { + key := cfgKludgeProfile + "." + strconv.Itoa(i) + "." + userAgent := v.GetString(key + "user_agent") + if userAgent == "" { + break + } + + kludgeParam := &KludgeParams{ + UseDefaultXMLNS: v.GetBool(cfgKludgeUseDefaultXMLNS), + BypassContentEncodingCheckInChunks: v.GetBool(cfgKludgeBypassContentEncodingCheckInChunks), + } + if v.IsSet(key + "use_default_xmlns") { + kludgeParam.UseDefaultXMLNS = v.GetBool(key + "use_default_xmlns") + } + + if v.IsSet(key + "bypass_content_encoding_check_in_chunks") { + kludgeParam.BypassContentEncodingCheckInChunks = v.GetBool(key + "bypass_content_encoding_check_in_chunks") + } + + kludgeProfiles[userAgent] = kludgeParam + } + + return kludgeProfiles +} + func fetchCopiesNumbers(l *zap.Logger, v *viper.Viper) map[string][]uint32 { copiesNums := make(map[string][]uint32) for i := 0; ; i++ { diff --git a/cmd/s3-gw/decoder_test.go b/cmd/s3-gw/decoder_test.go index 93827e6..50cd752 100644 --- a/cmd/s3-gw/decoder_test.go +++ b/cmd/s3-gw/decoder_test.go @@ -101,7 +101,7 @@ func TestDefaultNamespace(t *testing.T) { } { t.Run("", func(t *testing.T) { model := new(handler.CompleteMultipartUpload) - err := tc.settings.NewXMLDecoder(bytes.NewBufferString(tc.input)).Decode(model) + err := tc.settings.NewXMLDecoder(bytes.NewBufferString(tc.input), "test").Decode(model) if tc.err { require.Error(t, err) } else { diff --git a/config/config.env b/config/config.env index 44a8971..c556a52 100644 --- a/config/config.env +++ b/config/config.env @@ -186,6 +186,10 @@ S3_GW_KLUDGE_USE_DEFAULT_XMLNS=false S3_GW_KLUDGE_BYPASS_CONTENT_ENCODING_CHECK_IN_CHUNKS=false # Namespaces that should be handled as default S3_GW_KLUDGE_DEFAULT_NAMESPACES="" "root" +# Kludge profiles +S3_GW_KLUDGE_PROFILE_0_USER_AGENT=aws-cli +S3_GW_KLUDGE_PROFILE_0_USE_DEFAULT_XMLNS=true +S3_GW_KLUDGE_PROFILE_0_BYPASS_CONTENT_ENCODING_CHECK_IN_CHUNKS=true S3_GW_TRACING_ENABLED=false S3_GW_TRACING_ENDPOINT="localhost:4318" diff --git a/config/config.yaml b/config/config.yaml index 8f16946..98b706a 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -226,6 +226,13 @@ kludge: bypass_content_encoding_check_in_chunks: false # Namespaces that should be handled as default default_namespaces: [ "", "root" ] + # new profile section override defaults based on user agent + profile: + - user_agent: aws-cli + use_default_xmlns: false + - user_agent: aws-sdk-go + use_default_xmlns: true + bypass_content_encoding_check_in_chunks: false runtime: soft_memory_limit: 1gb diff --git a/docs/configuration.md b/docs/configuration.md index 0f98747..313c790 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -625,13 +625,19 @@ resolve_bucket: # `kludge` section -Workarounds for non-standard use cases. +Workarounds for non-standard use cases. In `profiles` subsection has the ability to override behavior for specific user agent. ```yaml kludge: use_default_xmlns: false bypass_content_encoding_check_in_chunks: false default_namespaces: [ "", "root" ] + profile: + - user_agent: aws-cli + use_default_xmlns: false + - user_agent: aws-sdk-go + use_default_xmlns: true + bypass_content_encoding_check_in_chunks: false ``` | Parameter | Type | SIGHUP reload | Default value | Description | @@ -639,6 +645,7 @@ kludge: | `use_default_xmlns` | `bool` | yes | `false` | Enable using default xml namespace `http://s3.amazonaws.com/doc/2006-03-01/` when parse xml bodies. | | `bypass_content_encoding_check_in_chunks` | `bool` | yes | `false` | Use this flag to be able to use [chunked upload approach](https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html) without having `aws-chunked` value in `Content-Encoding` header. | | `default_namespaces` | `[]string` | yes | `["","root"]` | Namespaces that should be handled as default. | +| `profile.user_agent` | `string` | yes | | Request UserAgent value. | # `runtime` section Contains runtime parameters.