From 40568590c73c0ece76afa6fd61cde73ecdd6f7e1 Mon Sep 17 00:00:00 2001 From: Roman Loginov Date: Thu, 31 Aug 2023 19:19:57 +0300 Subject: [PATCH] [#72] Support soft memory limit setting Signed-off-by: Roman Loginov --- app.go | 21 ++++++ config/config.env | 2 + config/config.yaml | 3 + docs/gate-configuration.md | 13 ++++ internal/logs/logs.go | 132 +++++++++++++++++++------------------ settings.go | 15 +++++ 6 files changed, 121 insertions(+), 65 deletions(-) diff --git a/app.go b/app.go index be9aaea..a0d6e0e 100644 --- a/app.go +++ b/app.go @@ -6,6 +6,7 @@ import ( "net/http" "os" "os/signal" + "runtime/debug" "sync" "syscall" "time" @@ -128,6 +129,8 @@ func newApp(ctx context.Context, opt ...Option) App { user.IDFromKey(&owner, a.key.PrivateKey.PublicKey) a.owner = &owner + a.setRuntimeParameters() + a.initAppSettings() a.initResolver() a.initMetrics() @@ -407,6 +410,8 @@ func (a *app) configReload(ctx context.Context) { a.log.Warn(logs.FailedToReloadServerParameters, zap.Error(err)) } + a.setRuntimeParameters() + a.stopServices() a.startServices() @@ -596,3 +601,19 @@ func (a *app) initTracing(ctx context.Context) { a.log.Info(logs.TracingConfigUpdated) } } + +func (a *app) setRuntimeParameters() { + if len(os.Getenv("GOMEMLIMIT")) != 0 { + // default limit < yaml limit < app env limit < GOMEMLIMIT + a.log.Warn(logs.RuntimeSoftMemoryDefinedWithGOMEMLIMIT) + return + } + + softMemoryLimit := fetchSoftMemoryLimit(a.cfg) + previous := debug.SetMemoryLimit(softMemoryLimit) + if softMemoryLimit != previous { + a.log.Info(logs.RuntimeSoftMemoryLimitUpdated, + zap.Int64("new_value", softMemoryLimit), + zap.Int64("old_value", previous)) + } +} diff --git a/config/config.env b/config/config.env index debdca2..62920a2 100644 --- a/config/config.env +++ b/config/config.env @@ -96,3 +96,5 @@ HTTP_GW_ZIP_COMPRESSION=false HTTP_GW_TRACING_ENABLED=true HTTP_GW_TRACING_ENDPOINT="localhost:4317" HTTP_GW_TRACING_EXPORTER="otlp_grpc" + +HTTP_GW_RUNTIME_SOFT_MEMORY_LIMIT=1073741824 diff --git a/config/config.yaml b/config/config.yaml index 510cb43..d2804d6 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -101,3 +101,6 @@ pool_error_threshold: 100 # The number of errors on connection after which node zip: compression: false # Enable zip compression to download files by common prefix. + +runtime: + soft_memory_limit: 1gb diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index 5b4edae..95e6c8e 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -53,6 +53,7 @@ $ cat http.log | `pprof` | [Pprof configuration](#pprof-section) | | `prometheus` | [Prometheus configuration](#prometheus-section) | | `tracing` | [Tracing configuration](#tracing-section) | +| `runtime` | [Runtime configuration](#runtime-section) | # General section @@ -256,3 +257,15 @@ tracing: | `enabled` | `bool` | yes | `false` | Flag to enable the tracing. | | `exporter` | `string` | yes | | Trace collector type (`stdout` or `otlp_grpc` are supported). | | `endpoint` | `string` | yes | | Address of collector endpoint for OTLP exporters. | + +# `runtime` section +Contains runtime parameters. + +```yaml +runtime: + soft_memory_limit: 1gb +``` + +| Parameter | Type | SIGHUP reload | Default value | Description | +|---------------------|--------|---------------|---------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `soft_memory_limit` | `size` | yes | maxint64 | Soft memory limit for the runtime. Zero or no value stands for no limit. If `GOMEMLIMIT` environment variable is set, the value from the configuration file will be ignored. | \ No newline at end of file diff --git a/internal/logs/logs.go b/internal/logs/logs.go index d47a2c3..ebb3c24 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -1,69 +1,71 @@ package logs const ( - CouldntParseCreationDate = "couldn't parse creation date" // Info in ../../downloader/* - CouldNotDetectContentTypeFromPayload = "could not detect Content-Type from payload" // Error in ../../downloader/download.go - CouldNotReceiveObject = "could not receive object" // Error in ../../downloader/download.go - WrongContainerID = "wrong container id" // Error in ../../downloader/download.go and uploader/upload.go - WrongObjectID = "wrong object id" // Error in ../../downloader/download.go - ObjectWasntFound = "object wasn't found" // Error in ../../downloader/download.go - ObjectWasDeleted = "object was deleted" // Error in ../../downloader/download.go - CouldNotSearchForObjects = "could not search for objects" // Error in ../../downloader/download.go - ObjectNotFound = "object not found" // Error in ../../downloader/download.go - ReadObjectListFailed = "read object list failed" // Error in ../../downloader/download.go - CouldNotCheckContainerExistence = "could not check container existence" // Error in ../../downloader/download.go - FailedToAddObjectToArchive = "failed to add object to archive" // Error in ../../downloader/download.go - IteratingOverSelectedObjectsFailed = "iterating over selected objects failed" // Error in ../../downloader/download.go - ObjectsNotFound = "objects not found" // Error in ../../downloader/download.go - CloseZipWriter = "close zip writer" // Error in ../../downloader/download.go - ServiceIsRunning = "service is running" // Info in ../../metrics/service.go - ServiceCouldntStartOnConfiguredPort = "service couldn't start on configured port" // Warn in ../../metrics/service.go - ServiceHasntStartedSinceItsDisabled = "service hasn't started since it's disabled" // Info in ../../metrics/service.go - ShuttingDownService = "shutting down service" // Info in ../../metrics/service.go - CantShutDownService = "can't shut down service" // Panic in ../../metrics/service.go - IgnorePartEmptyFormName = "ignore part, empty form name" // Debug in ../../uploader/upload.go - IgnorePartEmptyFilename = "ignore part, empty filename" // Debug in ../../uploader/upload.go - CloseTemporaryMultipartFormFile = "close temporary multipart/form file" // Debug in ../../uploader/upload.go - CouldNotReceiveMultipartForm = "could not receive multipart/form" // Error in ../../uploader/upload.go - CouldNotProcessHeaders = "could not process headers" // Error in ../../uploader/upload.go - CouldNotParseClientTime = "could not parse client time" // Warn in ../../uploader/upload.go - CouldNotPrepareExpirationHeader = "could not prepare expiration header" // Error in ../../uploader/upload.go - CouldNotEncodeResponse = "could not encode response" // Error in ../../uploader/upload.go - CouldNotStoreFileInFrostfs = "could not store file in frostfs" // Error in ../../uploader/upload.go - AddAttributeToResultObject = "add attribute to result object" // Debug in ../../uploader/filter.go - FailedToCreateResolver = "failed to create resolver" // Fatal in ../../app.go - ContainerResolverWillBeDisabledBecauseOfResolversResolverOrderIsEmpty = "container resolver will be disabled because of resolvers 'resolver_order' is empty" // Info in ../../app.go - MetricsAreDisabled = "metrics are disabled" // Warn in ../../app.go - NoWalletPathSpecifiedCreatingEphemeralKeyAutomaticallyForThisRun = "no wallet path specified, creating ephemeral key automatically for this run" // Info in ../../app.go - StartingApplication = "starting application" // Info in ../../app.go - StartingServer = "starting server" // Info in ../../app.go - ListenAndServe = "listen and serve" // Fatal in ../../app.go - ShuttingDownWebServer = "shutting down web server" // Info in ../../app.go - FailedToShutdownTracing = "failed to shutdown tracing" // Warn in ../../app.go - SIGHUPConfigReloadStarted = "SIGHUP config reload started" // Info in ../../app.go - FailedToReloadConfigBecauseItsMissed = "failed to reload config because it's missed" // Warn in ../../app.go - FailedToReloadConfig = "failed to reload config" // Warn in ../../app.go - LogLevelWontBeUpdated = "log level won't be updated" // Warn in ../../app.go - FailedToUpdateResolvers = "failed to update resolvers" // Warn in ../../app.go - FailedToReloadServerParameters = "failed to reload server parameters" // Warn in ../../app.go - SIGHUPConfigReloadCompleted = "SIGHUP config reload completed" // Info in ../../app.go - AddedPathUploadCid = "added path /upload/{cid}" // Info in ../../app.go - AddedPathGetCidOid = "added path /get/{cid}/{oid}" // Info in ../../app.go - AddedPathGetByAttributeCidAttrKeyAttrVal = "added path /get_by_attribute/{cid}/{attr_key}/{attr_val:*}" // Info in ../../app.go - AddedPathZipCidPrefix = "added path /zip/{cid}/{prefix}" // Info in ../../app.go - Request = "request" // Info in ../../app.go - CouldNotFetchAndStoreBearerToken = "could not fetch and store bearer token" // Error in ../../app.go - FailedToAddServer = "failed to add server" // Warn in ../../app.go - AddServer = "add server" // Info in ../../app.go - NoHealthyServers = "no healthy servers" // Fatal in ../../app.go - FailedToInitializeTracing = "failed to initialize tracing" // Warn in ../../app.go - TracingConfigUpdated = "tracing config updated" // Info in ../../app.go - ResolverNNSWontBeUsedSinceRPCEndpointIsntProvided = "resolver nns won't be used since rpc_endpoint isn't provided" // Warn in ../../app.go - CouldNotLoadFrostFSPrivateKey = "could not load FrostFS private key" // Fatal in ../../settings.go - UsingCredentials = "using credentials" // Info in ../../settings.go - FailedToCreateConnectionPool = "failed to create connection pool" // Fatal in ../../settings.go - FailedToDialConnectionPool = "failed to dial connection pool" // Fatal in ../../settings.go - FailedToCreateTreePool = "failed to create tree pool" // Fatal in ../../settings.go - FailedToDialTreePool = "failed to dial tree pool" // Fatal in ../../settings.go - AddedStoragePeer = "added storage peer" // Info in ../../settings.go + CouldntParseCreationDate = "couldn't parse creation date" // Info in ../../downloader/* + CouldNotDetectContentTypeFromPayload = "could not detect Content-Type from payload" // Error in ../../downloader/download.go + CouldNotReceiveObject = "could not receive object" // Error in ../../downloader/download.go + WrongContainerID = "wrong container id" // Error in ../../downloader/download.go and uploader/upload.go + WrongObjectID = "wrong object id" // Error in ../../downloader/download.go + ObjectWasntFound = "object wasn't found" // Error in ../../downloader/download.go + ObjectWasDeleted = "object was deleted" // Error in ../../downloader/download.go + CouldNotSearchForObjects = "could not search for objects" // Error in ../../downloader/download.go + ObjectNotFound = "object not found" // Error in ../../downloader/download.go + ReadObjectListFailed = "read object list failed" // Error in ../../downloader/download.go + CouldNotCheckContainerExistence = "could not check container existence" // Error in ../../downloader/download.go + FailedToAddObjectToArchive = "failed to add object to archive" // Error in ../../downloader/download.go + IteratingOverSelectedObjectsFailed = "iterating over selected objects failed" // Error in ../../downloader/download.go + ObjectsNotFound = "objects not found" // Error in ../../downloader/download.go + CloseZipWriter = "close zip writer" // Error in ../../downloader/download.go + ServiceIsRunning = "service is running" // Info in ../../metrics/service.go + ServiceCouldntStartOnConfiguredPort = "service couldn't start on configured port" // Warn in ../../metrics/service.go + ServiceHasntStartedSinceItsDisabled = "service hasn't started since it's disabled" // Info in ../../metrics/service.go + ShuttingDownService = "shutting down service" // Info in ../../metrics/service.go + CantShutDownService = "can't shut down service" // Panic in ../../metrics/service.go + IgnorePartEmptyFormName = "ignore part, empty form name" // Debug in ../../uploader/upload.go + IgnorePartEmptyFilename = "ignore part, empty filename" // Debug in ../../uploader/upload.go + CloseTemporaryMultipartFormFile = "close temporary multipart/form file" // Debug in ../../uploader/upload.go + CouldNotReceiveMultipartForm = "could not receive multipart/form" // Error in ../../uploader/upload.go + CouldNotProcessHeaders = "could not process headers" // Error in ../../uploader/upload.go + CouldNotParseClientTime = "could not parse client time" // Warn in ../../uploader/upload.go + CouldNotPrepareExpirationHeader = "could not prepare expiration header" // Error in ../../uploader/upload.go + CouldNotEncodeResponse = "could not encode response" // Error in ../../uploader/upload.go + CouldNotStoreFileInFrostfs = "could not store file in frostfs" // Error in ../../uploader/upload.go + AddAttributeToResultObject = "add attribute to result object" // Debug in ../../uploader/filter.go + FailedToCreateResolver = "failed to create resolver" // Fatal in ../../app.go + ContainerResolverWillBeDisabledBecauseOfResolversResolverOrderIsEmpty = "container resolver will be disabled because of resolvers 'resolver_order' is empty" // Info in ../../app.go + MetricsAreDisabled = "metrics are disabled" // Warn in ../../app.go + NoWalletPathSpecifiedCreatingEphemeralKeyAutomaticallyForThisRun = "no wallet path specified, creating ephemeral key automatically for this run" // Info in ../../app.go + StartingApplication = "starting application" // Info in ../../app.go + StartingServer = "starting server" // Info in ../../app.go + ListenAndServe = "listen and serve" // Fatal in ../../app.go + ShuttingDownWebServer = "shutting down web server" // Info in ../../app.go + FailedToShutdownTracing = "failed to shutdown tracing" // Warn in ../../app.go + SIGHUPConfigReloadStarted = "SIGHUP config reload started" // Info in ../../app.go + FailedToReloadConfigBecauseItsMissed = "failed to reload config because it's missed" // Warn in ../../app.go + FailedToReloadConfig = "failed to reload config" // Warn in ../../app.go + LogLevelWontBeUpdated = "log level won't be updated" // Warn in ../../app.go + FailedToUpdateResolvers = "failed to update resolvers" // Warn in ../../app.go + FailedToReloadServerParameters = "failed to reload server parameters" // Warn in ../../app.go + SIGHUPConfigReloadCompleted = "SIGHUP config reload completed" // Info in ../../app.go + AddedPathUploadCid = "added path /upload/{cid}" // Info in ../../app.go + AddedPathGetCidOid = "added path /get/{cid}/{oid}" // Info in ../../app.go + AddedPathGetByAttributeCidAttrKeyAttrVal = "added path /get_by_attribute/{cid}/{attr_key}/{attr_val:*}" // Info in ../../app.go + AddedPathZipCidPrefix = "added path /zip/{cid}/{prefix}" // Info in ../../app.go + Request = "request" // Info in ../../app.go + CouldNotFetchAndStoreBearerToken = "could not fetch and store bearer token" // Error in ../../app.go + FailedToAddServer = "failed to add server" // Warn in ../../app.go + AddServer = "add server" // Info in ../../app.go + NoHealthyServers = "no healthy servers" // Fatal in ../../app.go + FailedToInitializeTracing = "failed to initialize tracing" // Warn in ../../app.go + TracingConfigUpdated = "tracing config updated" // Info in ../../app.go + ResolverNNSWontBeUsedSinceRPCEndpointIsntProvided = "resolver nns won't be used since rpc_endpoint isn't provided" // Warn in ../../app.go + RuntimeSoftMemoryDefinedWithGOMEMLIMIT = "soft runtime memory defined with GOMEMLIMIT environment variable, config value skipped" // Warn in ../../app.go + RuntimeSoftMemoryLimitUpdated = "soft runtime memory limit value updated" // Info in ../../app.go + CouldNotLoadFrostFSPrivateKey = "could not load FrostFS private key" // Fatal in ../../settings.go + UsingCredentials = "using credentials" // Info in ../../settings.go + FailedToCreateConnectionPool = "failed to create connection pool" // Fatal in ../../settings.go + FailedToDialConnectionPool = "failed to dial connection pool" // Fatal in ../../settings.go + FailedToCreateTreePool = "failed to create tree pool" // Fatal in ../../settings.go + FailedToDialTreePool = "failed to dial tree pool" // Fatal in ../../settings.go + AddedStoragePeer = "added storage peer" // Info in ../../settings.go ) diff --git a/settings.go b/settings.go index ed13da2..6708ad4 100644 --- a/settings.go +++ b/settings.go @@ -4,6 +4,7 @@ import ( "context" "encoding/hex" "fmt" + "math" "os" "path" "runtime" @@ -36,6 +37,8 @@ const ( defaultPoolErrorThreshold uint32 = 100 + defaultSoftMemoryLimit = math.MaxInt64 + cfgServer = "server" cfgTLSEnabled = "tls.enabled" cfgTLSCertFile = "tls.cert_file" @@ -90,6 +93,9 @@ const ( // Zip compression. cfgZipCompression = "zip.compression" + // Runtime. + cfgSoftMemoryLimit = "runtime.soft_memory_limit" + // Command line args. cmdHelp = "help" cmdVersion = "version" @@ -516,3 +522,12 @@ func fetchPeers(l *zap.Logger, v *viper.Viper) []pool.NodeParam { return nodes } + +func fetchSoftMemoryLimit(cfg *viper.Viper) int64 { + softMemoryLimit := cfg.GetSizeInBytes(cfgSoftMemoryLimit) + if softMemoryLimit <= 0 { + softMemoryLimit = defaultSoftMemoryLimit + } + + return int64(softMemoryLimit) +}