From 0f3ee73f2cba0cdba3857f3a4585a180c57eb99f Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 25 Aug 2023 14:53:59 +0300 Subject: [PATCH] [#70] Support configuring buffer size for put Signed-off-by: Denis Kirillov --- CHANGELOG.md | 2 +- cmd/http-gw/app.go | 22 ++++++++++++++++++---- cmd/http-gw/settings.go | 7 +++++++ config/config.env | 2 ++ config/config.yaml | 2 ++ docs/gate-configuration.md | 8 +++++--- internal/handler/handler.go | 1 + internal/handler/upload.go | 1 + 8 files changed, 37 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d660f10..10d42c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ This document outlines major changes between releases. - Support impersonate bearer token (#40, #45) - Tracing support (#20, #44, #60) - Object name resolving with tree service (#30) -- Add new `frostfs.client_cut` config param (#70) +- Add new `frostfs.client_cut` and `frostfs.buffer_max_size_for_put` config params (#70) ### Changed - Update prometheus to v1.15.0 (#35) diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index 93a6b6c..81bfb14 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -72,10 +72,11 @@ type ( // appSettings stores reloading parameters, so it has to provide getters and setters which use RWMutex. appSettings struct { - mu sync.RWMutex - defaultTimestamp bool - zipCompression bool - clientCut bool + mu sync.RWMutex + defaultTimestamp bool + zipCompression bool + clientCut bool + bufferMaxSizeForPut uint64 } ) @@ -177,6 +178,18 @@ func (s *appSettings) setClientCut(val bool) { s.mu.Unlock() } +func (s *appSettings) BufferMaxSizeForPut() uint64 { + s.mu.RLock() + defer s.mu.RUnlock() + return s.bufferMaxSizeForPut +} + +func (s *appSettings) setBufferMaxSizeForPut(val uint64) { + s.mu.Lock() + s.bufferMaxSizeForPut = val + s.mu.Unlock() +} + func (a *app) initAppSettings() { a.settings = &appSettings{} @@ -462,6 +475,7 @@ func (a *app) updateSettings() { a.settings.setDefaultTimestamp(a.cfg.GetBool(cfgUploaderHeaderEnableDefaultTimestamp)) a.settings.setZipCompression(a.cfg.GetBool(cfgZipCompression)) a.settings.setClientCut(a.cfg.GetBool(cfgClientCut)) + a.settings.setBufferMaxSizeForPut(a.cfg.GetUint64(cfgBufferMaxSizeForPut)) } func (a *app) startServices() { diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index 86718eb..b097aa3 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -39,6 +39,8 @@ const ( defaultSoftMemoryLimit = math.MaxInt64 + defaultBufferMaxSizeForPut = 1024 * 1024 // 1mb + cfgServer = "server" cfgTLSEnabled = "tls.enabled" cfgTLSCertFile = "tls.cert_file" @@ -98,6 +100,8 @@ const ( // Enabling client side object preparing for PUT operations. cfgClientCut = "frostfs.client_cut" + // Sets max buffer size for read payload in put operations. + cfgBufferMaxSizeForPut = "frostfs.buffer_max_size_for_put" // Command line args. cmdHelp = "help" @@ -160,6 +164,9 @@ func settings() *viper.Viper { // pool: v.SetDefault(cfgPoolErrorThreshold, defaultPoolErrorThreshold) + // frostfs: + v.SetDefault(cfgBufferMaxSizeForPut, defaultBufferMaxSizeForPut) + // web-server: v.SetDefault(cfgWebReadBufferSize, 4096) v.SetDefault(cfgWebWriteBufferSize, 4096) diff --git a/config/config.env b/config/config.env index 58e6814..06e71f8 100644 --- a/config/config.env +++ b/config/config.env @@ -102,3 +102,5 @@ HTTP_GW_RUNTIME_SOFT_MEMORY_LIMIT=1073741824 # Parameters of requests to FrostFS # This flag enables client side object preparing. HTTP_GW_FROSTFS_CLIENT_CUT=false +# Sets max buffer size for read payload in put operations. +HTTP_GW_FROSTFS_BUFFER_MAX_SIZE_FOR_PUT=1048576 diff --git a/config/config.yaml b/config/config.yaml index 3eae752..15afeec 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -109,3 +109,5 @@ runtime: frostfs: # This flag enables client side object preparing. client_cut: false + # Sets max buffer size for read payload in put operations. + buffer_max_size_for_put: 1048576 diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index 25d068b..852908b 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -278,8 +278,10 @@ Contains parameters of requests to FrostFS. ```yaml frostfs: client_cut: false + buffer_max_size_for_put: 1048576 # 1mb ``` -| Parameter | Type | SIGHUP reload | Default value | Description | -|--------------|--------|---------------|---------------|-------------------------------------------------| -| `client_cut` | `bool` | yes | `false` | This flag enables client side object preparing. | +| Parameter | Type | SIGHUP reload | Default value | Description | +|---------------------------|----------|---------------|---------------|----------------------------------------------------------| +| `client_cut` | `bool` | yes | `false` | This flag enables client side object preparing. | +| `buffer_max_size_for_put` | `uint64` | yes | `1048576` | Sets max buffer size for read payload in put operations. | diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 2bb4347..61e2bc1 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -24,6 +24,7 @@ type Config interface { DefaultTimestamp() bool ZipCompression() bool ClientCut() bool + BufferMaxSizeForPut() uint64 } type Handler struct { diff --git a/internal/handler/upload.go b/internal/handler/upload.go index a8c4365..b95896f 100644 --- a/internal/handler/upload.go +++ b/internal/handler/upload.go @@ -137,6 +137,7 @@ func (h *Handler) Upload(req *fasthttp.RequestCtx) { prm.SetHeader(*obj) prm.SetPayload(file) prm.SetClientCut(h.config.ClientCut()) + prm.SetBufferMaxSize(h.config.BufferMaxSizeForPut()) bt := h.fetchBearerToken(ctx) if bt != nil {