[#369] Write log to file

Signed-off-by: Nikita Zinkevich <n.zinkevich@yadro.com>

api/middleware/constants.go

@@ -106,3 +106,9 @@ const (
 	PartNumberQuery     = "partNumber"
 	LegalHoldQuery      = "legal-hold"
 )
+
+const (
+	StdoutPath = "stdout"
+	StderrPath = "stderr"
+	SinkName   = "lumberjack"
+)

api/middleware/log_http.go

@@ -1,27 +1,81 @@
 package middleware
 
 import (
+	"bytes"
+	"fmt"
 	"io"
 	"net/http"
+	"net/url"
 
 	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
 	"go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
+	"gopkg.in/natefinch/lumberjack.v2"
 )
 
-type LogHTTPSettings struct {
+type LogHTTPConfig struct {
-	Enabled bool
+	Enabled    bool
-	MaxBody int64
+	MaxBody    int64
+	MaxLogSize int
+	OutputPath string
+	UseGzip    bool
+}
+
+// getFileLogger returns file logger for http requests or creates new if not exists.
+func (h *LogHTTPConfig) getFileLogger() (*zap.Logger, error) {
+	c := newLoggerConfig()
+	c.OutputPaths = []string{h.OutputPath}
+	if h.OutputPath != StdoutPath && h.OutputPath != StderrPath && h.OutputPath != "" {
+		err := h.registerOutputSink()
+		if err != nil {
+			return nil, err
+		}
+		c.OutputPaths[0] = SinkName + ":" + h.OutputPath
+	}
+
+	return c.Build()
+}
+
+// registerOutputSink registers sink for logger file output.
+func (h *LogHTTPConfig) registerOutputSink() error {
+	logRoller := &lumberjack.Logger{
+		Filename: h.OutputPath,
+		MaxSize:  h.MaxLogSize,
+		Compress: h.UseGzip,
+	}
+	err := zap.RegisterSink(SinkName, func(_ *url.URL) (zap.Sink, error) {
+		return lumberjackSink{
+			Logger: logRoller,
+		}, nil
+	})
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+type lumberjackSink struct {
+	*lumberjack.Logger
+}
+
+func (lumberjackSink) Sync() error {
+	return nil
 }
 
 // LogHTTP logs http parameters from s3 request.
-func LogHTTP(l *zap.Logger, settings LogHTTPSettings) Func {
+func LogHTTP(l *zap.Logger, config LogHTTPConfig) Func {
+	fileLogger, err := config.getFileLogger()
+	if err != nil {
+		return func(h http.Handler) http.Handler {
+			return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				h.ServeHTTP(w, r)
+			})
+		}
+	}
 	return func(h http.Handler) http.Handler {
 		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			if !settings.Enabled {
+			var httplog = fileLogger.With(
-				h.ServeHTTP(w, r)
-				return
-			}
-			var httplog = l.With(
 				zap.String("from", r.RemoteAddr),
 				zap.String("URI", r.RequestURI),
 				zap.String("method", r.Method),
@@ -29,11 +83,11 @@ func LogHTTP(l *zap.Logger, settings LogHTTPSettings) Func {
 
 			httplog = withFieldIfExist(httplog, "query", r.URL.Query())
 			httplog = withFieldIfExist(httplog, "headers", r.Header)
-			if r.ContentLength != 0 && r.ContentLength <= settings.MaxBody {
+			if r.ContentLength > 0 && r.ContentLength <= config.MaxBody {
 				var err error
 				httplog, err = withBody(httplog, r)
 				if err != nil {
-					l.Error("read body error")
+					l.Error("read body error", zap.Error(err))
 				}
 			}
 			httplog.Info(logs.RequestHTTP)
@@ -42,17 +96,34 @@ func LogHTTP(l *zap.Logger, settings LogHTTPSettings) Func {
 	}
 }
 
+// newLoggerConfig creates new zap.Config with disabled base fields.
+func newLoggerConfig() zap.Config {
+	c := zap.NewProductionConfig()
+	c.DisableCaller = true
+	c.DisableStacktrace = true
+	c.EncoderConfig.MessageKey = zapcore.OmitKey
+	c.EncoderConfig.LevelKey = zapcore.OmitKey
+	c.EncoderConfig.TimeKey = zapcore.OmitKey
+	c.EncoderConfig.FunctionKey = zapcore.OmitKey
+
+	return c
+}
+
+// withBody reads body and attach it to log output.
 func withBody(httplog *zap.Logger, r *http.Request) (*zap.Logger, error) {
-	var body = make([]byte, r.ContentLength)
+	body, err := io.ReadAll(r.Body)
-	_, err := r.Body.Read(body)
+	if err != nil {
-	if err != nil && err != io.EOF {
+		return nil, fmt.Errorf("read body error: %w", err)
-		return nil, err
 	}
+	defer r.Body.Close()
+	r.Body = io.NopCloser(bytes.NewBuffer(body))
+
 	httplog = httplog.With(zap.String("body", string(body)))
 
 	return httplog, nil
 }
 
+// withFieldIfExist checks whether data is not empty and attach it to log output.
 func withFieldIfExist(log *zap.Logger, label string, data map[string][]string) *zap.Logger {
 	if len(data) != 0 {
 		log = log.With(zap.Any(label, data))

api/router.go

@@ -109,7 +109,7 @@ type Config struct {
 	Handler  Handler
 	Center   s3middleware.Center
 	Log      *zap.Logger
-	LogHTTP  s3middleware.LogHTTPSettings
+	LogHTTP  s3middleware.LogHTTPConfig
 	Metrics  *metrics.AppMetrics
 
 	MiddlewareSettings Settings
@@ -129,8 +129,12 @@ type Config struct {
 
 func NewRouter(cfg Config) *chi.Mux {
 	api := chi.NewRouter()
+
+	if cfg.LogHTTP.Enabled {
+		api.Use(s3middleware.LogHTTP(cfg.Log, cfg.LogHTTP))
+	}
+
 	api.Use(
-		s3middleware.LogHTTP(cfg.Log, cfg.LogHTTP),
 		s3middleware.Request(cfg.Log, cfg.MiddlewareSettings),
 		middleware.ThrottleWithOpts(cfg.Throttle),
 		middleware.Recoverer,

cmd/s3-gw/app.go

@@ -83,8 +83,7 @@ type (
 
 	appSettings struct {
 		logLevel            zap.AtomicLevel
-		httpLoggingEnabled  bool
+		httpLogging         s3middleware.LogHTTPConfig
-		maxHTTPLogBody      int64
 		maxClient           maxClientsConfig
 		defaultMaxAge       int
 		reconnectInterval   time.Duration
@@ -195,8 +194,6 @@ func newAppSettings(log *Logger, v *viper.Viper) *appSettings {
 		defaultMaxAge:       fetchDefaultMaxAge(v, log.logger),
 		reconnectInterval:   fetchReconnectInterval(v),
 		frostfsidValidation: v.GetBool(cfgFrostfsIDValidationEnabled),
-		httpLoggingEnabled:  v.GetBool(cfgLoggerRequestEnabled),
-		maxHTTPLogBody:      v.GetInt64(cfgLoggerRequestMaxBody),
 	}
 
 	settings.resolveZoneList = v.GetStringSlice(cfgResolveBucketAllow)
@@ -222,6 +219,7 @@ func (s *appSettings) update(v *viper.Viper, log *zap.Logger) {
 	s.setRetryMaxAttempts(fetchRetryMaxAttempts(v))
 	s.setRetryMaxBackoff(fetchRetryMaxBackoff(v))
 	s.setRetryStrategy(fetchRetryStrategy(v))
+	s.updateHTTPLoggingSettings(v)
 }
 
 func (s *appSettings) updateNamespacesSettings(v *viper.Viper, log *zap.Logger) {
@@ -565,6 +563,17 @@ func newMaxClients(cfg *viper.Viper) maxClientsConfig {
 	return config
 }
 
+func (s *appSettings) updateHTTPLoggingSettings(cfg *viper.Viper) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	s.httpLogging.Enabled = cfg.GetBool(cfgHTTPLoggingEnabled)
+	s.httpLogging.MaxBody = cfg.GetInt64(cfgHTTPLoggingMaxBody)
+	s.httpLogging.MaxLogSize = cfg.GetInt(cfgHTTPLoggingMaxLogSize)
+	s.httpLogging.OutputPath = cfg.GetString(cfgHTTPLoggingDestination)
+	s.httpLogging.UseGzip = cfg.GetBool(cfgHTTPLoggingGzip)
+}
+
 func getPools(ctx context.Context, logger *zap.Logger, cfg *viper.Viper) (*pool.Pool, *treepool.Pool, *keys.PrivateKey) {
 	var prm pool.InitParameters
 	var prmTree treepool.InitParameters
@@ -686,10 +695,7 @@ func (a *App) Serve(ctx context.Context) {
 		Handler: a.api,
 		Center:  a.ctr,
 		Log:     a.log,
-		LogHTTP: s3middleware.LogHTTPSettings{
+		LogHTTP: a.settings.httpLogging,
-			Enabled: a.settings.httpLoggingEnabled,
-			MaxBody: a.settings.maxHTTPLogBody,
-		},
 		Metrics: a.metrics,
 		Domains: domains,
 

cmd/s3-gw/app_settings.go

@@ -75,8 +75,12 @@ const ( // Settings.
 	cfgLoggerLevel       = "logger.level"
 	cfgLoggerDestination = "logger.destination"
 
-	cfgLoggerRequestEnabled = "http_logging.enabled"
+	// HttpLogging.
-	cfgLoggerRequestMaxBody = "http_logging.max_body"
+	cfgHTTPLoggingEnabled     = "http_logging.enabled"
+	cfgHTTPLoggingMaxBody     = "http_logging.max_body"
+	cfgHTTPLoggingMaxLogSize  = "http_logging.max_log_size"
+	cfgHTTPLoggingDestination = "http_logging.destination"
+	cfgHTTPLoggingGzip        = "http_logging.gzip"
 
 	// Wallet.
 	cfgWalletPath       = "wallet.path"
@@ -719,8 +723,11 @@ func newSettings() *viper.Viper {
 	v.SetDefault(cfgLoggerDestination, "stdout")
 
 	// http logger
-	v.SetDefault(cfgLoggerRequestEnabled, false)
+	v.SetDefault(cfgHTTPLoggingEnabled, false)
-	v.SetDefault(cfgLoggerRequestMaxBody, 1024)
+	v.SetDefault(cfgHTTPLoggingMaxBody, 1024)
+	v.SetDefault(cfgHTTPLoggingMaxLogSize, 50)
+	v.SetDefault(cfgHTTPLoggingDestination, "stdout")
+	v.SetDefault(cfgHTTPLoggingGzip, false)
 
 	// pool:
 	v.SetDefault(cfgPoolErrorThreshold, defaultPoolErrorThreshold)

config/config.env

@@ -45,6 +45,17 @@ S3_GW_CONFIG=/path/to/config/yaml
 # Logger
 S3_GW_LOGGER_LEVEL=debug
 
+# HTTP logger
+S3_GW_HTTP_LOGGING_ENABLED=false
+# max body size to log
+S3_GW_HTTP_LOGGING_MAX_BODY=1024
+# max log size in Mb
+S3_GW_HTTP_LOGGING_MAX_LOG_SIZE: 20
+# use log compression
+S3_GW_HTTP_LOGGING_GZIP=true
+# possible destination output values: filesystem path, url, "stdout", "stderr"
+S3_GW_HTTP_LOGGING_DESTINATION=stdout
+
 # RPC endpoint and order of resolving of bucket names
 S3_GW_RPC_ENDPOINT=http://morph-chain.frostfs.devenv:30333/
 S3_GW_RESOLVE_ORDER="nns dns"

config/config.yaml

@@ -52,6 +52,12 @@ http_logging:
   enabled: false
   # max body size to log
   max_body: 1024
+  # max log size in Mb
+  max_log_size: 20
+  # use log compression
+  gzip: true
+  # possible output values: filesystem path, url, "stdout", "stderr"
+  destination: stdout
   
 # RPC endpoint and order of resolving of bucket names
 rpc_endpoint: http://morph-chain.frostfs.devenv:30333

docs/configuration.md

@@ -176,6 +176,7 @@ There are some custom types used for brevity:
 | `placement_policy` | [Placement policy configuration](#placement_policy-section)    |
 | `server`           | [Server configuration](#server-section)                        |
 | `logger`           | [Logger configuration](#logger-section)                        |
+| `http_logging`      | [HTTP Request logger configuration](#http_logging-section)     |
 | `cache`            | [Cache configuration](#cache-section)                          |
 | `cors`             | [CORS configuration](#cors-section)                            |
 | `pprof`            | [Pprof configuration](#pprof-section)                          |
@@ -373,6 +374,27 @@ logger:
 | `level`       | `string` | yes           | `debug`       | Logging level.<br/>Possible values:  `debug`, `info`, `warn`, `error`, `dpanic`, `panic`, `fatal`. |
 | `destination` | `string` | no            | `stdout`      | Destination for logger: `stdout` or `journald`                                                     |
 
+
+### `http_logging` section
+
+```yaml
+http_logging:
+  enabled: false
+  max_body: 1024
+  max_log_size: 20
+  gzip: true
+  destination: stdout
+```
+
+| Parameter      | Type   | SIGHUP reload | Default value | Description                                                                                                                                                                                 |
+|----------------|--------|---------------|---------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `enabled`      | bool   | yes           | false         | Flag to enable the logger.                                                                                                                                                                  |
+| `max_body`     | int    | yes           | 1024          | Max body size for log output in bytes.                                                                                                                                                      |
+| `max_log_size` | int    | no            | 50            | Log file size threshold (in megabytes) to be moved in backup file. After reaching  threshold, initial filename is appended with timestamp. And new empty file with initial name is created. |
+| `gzip`         | bool   | no            | false         | Whether to enable Gzip compression for backup log files.                                                                                                                                    |
+| `destination`  | string | no            | stdout        | Specify path for log output. Accepts path string to log in file, or "stdout" and "stderr" reserved phrases to print in output streams.                                                      |
+
+
 ### `cache` section
 
 ```yaml

go.mod

@@ -34,6 +34,7 @@ require (
 	golang.org/x/text v0.14.0
 	google.golang.org/grpc v1.59.0
 	google.golang.org/protobuf v1.33.0
+	gopkg.in/natefinch/lumberjack.v2 v2.2.1
 )
 
 require (

go.sum

@@ -696,6 +696,8 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
 gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
+gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

internal/logs/logs.go

@@ -100,48 +100,47 @@ const (
 	FailedToPassAuthentication                           = "failed to pass authentication"                                                                     // Error in ../../api/middleware/auth.go
 	FailedToResolveCID                                   = "failed to resolve CID"                                                                             // Debug in ../../api/middleware/metrics.go
 	RequestStart                                         = "request start"                                                                                     // Info in ../../api/middleware/reqinfo.go
-
+	RequestHTTP                                          = "http request"
-	RequestHTTP                                    = "http request"
+	FailedToUnescapeObjectName                           = "failed to unescape object name"                                                         // Warn in ../../api/middleware/reqinfo.go
-	FailedToUnescapeObjectName                     = "failed to unescape object name"                                                         // Warn in ../../api/middleware/reqinfo.go
+	InvalidDefaultMaxAge                                 = "invalid defaultMaxAge"                                                                  // Fatal in ../../cmd/s3-gw/app_settings.go
-	InvalidDefaultMaxAge                           = "invalid defaultMaxAge"                                                                  // Fatal in ../../cmd/s3-gw/app_settings.go
+	CantShutDownService                                  = "can't shut down service"                                                                // Panic in ../../cmd/s3-gw/service.go
-	CantShutDownService                            = "can't shut down service"                                                                // Panic in ../../cmd/s3-gw/service.go
+	CouldntGenerateRandomKey                             = "couldn't generate random key"                                                           // Fatal in ../../cmd/s3-gw/app.go
-	CouldntGenerateRandomKey                       = "couldn't generate random key"                                                           // Fatal in ../../cmd/s3-gw/app.go
+	FailedToCreateResolver                               = "failed to create resolver"                                                              // Fatal in ../../cmd/s3-gw/app.go
-	FailedToCreateResolver                         = "failed to create resolver"                                                              // Fatal in ../../cmd/s3-gw/app.go
+	CouldNotLoadFrostFSPrivateKey                        = "could not load FrostFS private key"                                                     // Fatal in ../../cmd/s3-gw/app.go
-	CouldNotLoadFrostFSPrivateKey                  = "could not load FrostFS private key"                                                     // Fatal in ../../cmd/s3-gw/app.go
+	FailedToCreateConnectionPool                         = "failed to create connection pool"                                                       // Fatal in ../../cmd/s3-gw/app.go
-	FailedToCreateConnectionPool                   = "failed to create connection pool"                                                       // Fatal in ../../cmd/s3-gw/app.go
+	FailedToDialConnectionPool                           = "failed to dial connection pool"                                                         // Fatal in ../../cmd/s3-gw/app.go
-	FailedToDialConnectionPool                     = "failed to dial connection pool"                                                         // Fatal in ../../cmd/s3-gw/app.go
+	FailedToCreateTreePool                               = "failed to create tree pool"                                                             // Fatal in ../../cmd/s3-gw/app.go
-	FailedToCreateTreePool                         = "failed to create tree pool"                                                             // Fatal in ../../cmd/s3-gw/app.go
+	FailedToDialTreePool                                 = "failed to dial tree pool"                                                               // Fatal in ../../cmd/s3-gw/app.go
-	FailedToDialTreePool                           = "failed to dial tree pool"                                                               // Fatal in ../../cmd/s3-gw/app.go
+	ListenAndServe                                       = "listen and serve"                                                                       // Fatal in ../../cmd/s3-gw/app.go
-	ListenAndServe                                 = "listen and serve"                                                                       // Fatal in ../../cmd/s3-gw/app.go
+	NoHealthyServers                                     = "no healthy servers"                                                                     // Fatal in ../../cmd/s3-gw/app.go
-	NoHealthyServers                               = "no healthy servers"                                                                     // Fatal in ../../cmd/s3-gw/app.go
+	CouldNotInitializeAPIHandler                         = "could not initialize API handler"                                                       // Fatal in ../../cmd/s3-gw/app.go
-	CouldNotInitializeAPIHandler                   = "could not initialize API handler"                                                       // Fatal in ../../cmd/s3-gw/app.go
+	RuntimeSoftMemoryDefinedWithGOMEMLIMIT               = "soft runtime memory defined with GOMEMLIMIT environment variable, config value skipped" // Warn in ../../cmd/s3-gw/app.go
-	RuntimeSoftMemoryDefinedWithGOMEMLIMIT         = "soft runtime memory defined with GOMEMLIMIT environment variable, config value skipped" // Warn in ../../cmd/s3-gw/app.go
+	RuntimeSoftMemoryLimitUpdated                        = "soft runtime memory limit value updated"                                                // Info in ../../cmd/s3-gw/app.go
-	RuntimeSoftMemoryLimitUpdated                  = "soft runtime memory limit value updated"                                                // Info in ../../cmd/s3-gw/app.go
+	AnonRequestSkipFrostfsIDValidation                   = "anon request, skip FrostfsID validation"                                                // Debug in ../../api/middleware/auth.go
-	AnonRequestSkipFrostfsIDValidation             = "anon request, skip FrostfsID validation"                                                // Debug in ../../api/middleware/auth.go
+	FrostfsIDValidationFailed                            = "FrostfsID validation failed"                                                            // Error in ../../api/middleware/auth.go
-	FrostfsIDValidationFailed                      = "FrostfsID validation failed"                                                            // Error in ../../api/middleware/auth.go
+	InitFrostfsIDContractFailed                          = "init frostfsid contract failed"                                                         // Fatal in ../../cmd/s3-gw/app.go
-	InitFrostfsIDContractFailed                    = "init frostfsid contract failed"                                                         // Fatal in ../../cmd/s3-gw/app.go
+	InitPolicyContractFailed                             = "init policy contract failed"                                                            // Fatal in ../../cmd/s3-gw/app.go
-	InitPolicyContractFailed                       = "init policy contract failed"                                                            // Fatal in ../../cmd/s3-gw/app.go
+	PolicyValidationFailed                               = "policy validation failed"
-	PolicyValidationFailed                         = "policy validation failed"
+	ServerReconnecting                                   = "reconnecting server..."
-	ServerReconnecting                             = "reconnecting server..."
+	ServerReconnectedSuccessfully                        = "server reconnected successfully"
-	ServerReconnectedSuccessfully                  = "server reconnected successfully"
+	ServerReconnectFailed                                = "failed to reconnect server"
-	ServerReconnectFailed                          = "failed to reconnect server"
+	ParseTreeNode                                        = "parse tree node"
-	ParseTreeNode                                  = "parse tree node"
+	FailedToGetRealObjectSize                            = "failed to get real object size"
-	FailedToGetRealObjectSize                      = "failed to get real object size"
+	CouldntDeleteObjectFromStorageContinueDeleting       = "couldn't delete object from storage, continue deleting from tree"
-	CouldntDeleteObjectFromStorageContinueDeleting = "couldn't delete object from storage, continue deleting from tree"
+	CouldntPutAccessBoxIntoCache                         = "couldn't put accessbox into cache"
-	CouldntPutAccessBoxIntoCache                   = "couldn't put accessbox into cache"
+	InvalidAccessBoxCacheRemovingCheckInterval           = "invalid accessbox check removing interval, using default value"
-	InvalidAccessBoxCacheRemovingCheckInterval     = "invalid accessbox check removing interval, using default value"
+	CouldNotCloseRequestBody                             = "could not close request body"
-	CouldNotCloseRequestBody                       = "could not close request body"
+	BucketOwnerKeyIsMissing                              = "bucket owner key is missing"
-	BucketOwnerKeyIsMissing                        = "bucket owner key is missing"
+	SettingsNodeInvalidOwnerKey                          = "settings node: invalid owner key"
-	SettingsNodeInvalidOwnerKey                    = "settings node: invalid owner key"
+	SuccessfulAuth                                       = "successful auth"
-	SuccessfulAuth                                 = "successful auth"
+	PolicyRequest                                        = "policy request"
-	PolicyRequest                                  = "policy request"
+	FailedToGenerateRequestID                            = "failed to generate request id"
-	FailedToGenerateRequestID                      = "failed to generate request id"
+	InvalidBucketObjectLockEnabledHeader                 = "invalid X-Amz-Bucket-Object-Lock-Enabled header"
-	InvalidBucketObjectLockEnabledHeader           = "invalid X-Amz-Bucket-Object-Lock-Enabled header"
+	InvalidTreeKV                                        = "invalid tree service meta KV"
-	InvalidTreeKV                                  = "invalid tree service meta KV"
+	FailedToWriteResponse                                = "failed to write response"
-	FailedToWriteResponse                          = "failed to write response"
+	WarnDuplicateAddress                                 = "duplicate address"
-	WarnDuplicateAddress                           = "duplicate address"
+	PolicyCouldntBeConvertedToNativeRules                = "policy couldn't be converted to native rules, only s3 rules be applied"
-	PolicyCouldntBeConvertedToNativeRules          = "policy couldn't be converted to native rules, only s3 rules be applied"
+	CouldntCacheSubject                                  = "couldn't cache subject info"
-	CouldntCacheSubject                            = "couldn't cache subject info"
+	UserGroupsListIsEmpty                                = "user groups list is empty, subject not found"
-	UserGroupsListIsEmpty                          = "user groups list is empty, subject not found"
+	CouldntCacheUserKey                                  = "couldn't cache user key"
-	CouldntCacheUserKey                            = "couldn't cache user key"
 )