diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index 1bacbed..103c72b 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -44,6 +44,7 @@ import ( "github.com/valyala/fasthttp" "go.opentelemetry.io/otel/trace" "go.uber.org/zap" + "go.uber.org/zap/zapcore" "golang.org/x/exp/slices" ) @@ -51,7 +52,6 @@ type ( app struct { ctx context.Context log *zap.Logger - logLevel zap.AtomicLevel pool *pool.Pool treePool *treepool.Pool key *keys.PrivateKey @@ -94,6 +94,7 @@ type ( reconnectInterval time.Duration dialerSource *internalnet.DialerSource workerPoolSize int + logLevelConfig *logLevelConfig mu sync.RWMutex defaultTimestamp bool @@ -113,6 +114,15 @@ type ( enableFilepathFallback bool } + tagsConfig struct { + tagLogs sync.Map + } + + logLevelConfig struct { + logLevel zap.AtomicLevel + tagsConfig *tagsConfig + } + CORS struct { AllowOrigin string AllowMethods []string @@ -123,14 +133,91 @@ type ( } ) +func newLogLevel(v *viper.Viper) zap.AtomicLevel { + ll, err := getLogLevel(v) + if err != nil { + panic(err.Error()) + } + atomicLogLevel := zap.NewAtomicLevel() + atomicLogLevel.SetLevel(ll) + return atomicLogLevel +} + +func newTagsConfig(v *viper.Viper, ll zapcore.Level) *tagsConfig { + var t tagsConfig + if err := t.update(v, ll); err != nil { + // panic here is analogue of the similar panic during common log level initialization. + panic(err.Error()) + } + return &t +} + +func newLogLevelConfig(lvl zap.AtomicLevel, tagsConfig *tagsConfig) *logLevelConfig { + return &logLevelConfig{ + logLevel: lvl, + tagsConfig: tagsConfig, + } +} + +func (l *logLevelConfig) update(cfg *viper.Viper, log *zap.Logger) { + if lvl, err := getLogLevel(cfg); err != nil { + log.Warn(logs.LogLevelWontBeUpdated, zap.Error(err), logs.TagField(logs.TagApp)) + } else { + l.logLevel.SetLevel(lvl) + } + + if err := l.tagsConfig.update(cfg, l.logLevel.Level()); err != nil { + log.Warn(logs.TagsLogConfigWontBeUpdated, zap.Error(err), logs.TagField(logs.TagApp)) + } +} + +func (t *tagsConfig) LevelEnabled(tag string, tgtLevel zapcore.Level) bool { + lvl, ok := t.tagLogs.Load(tag) + if !ok { + return false + } + + return lvl.(zapcore.Level).Enabled(tgtLevel) +} + +func (t *tagsConfig) update(cfg *viper.Viper, ll zapcore.Level) error { + tags, err := fetchLogTagsConfig(cfg, ll) + if err != nil { + return err + } + + t.tagLogs.Range(func(key, value any) bool { + k := key.(string) + v := value.(zapcore.Level) + + if lvl, ok := tags[k]; ok { + if lvl != v { + t.tagLogs.Store(key, lvl) + } + } else { + t.tagLogs.Delete(key) + delete(tags, k) + } + return true + }) + + for k, v := range tags { + t.tagLogs.Store(k, v) + } + + return nil +} + func newApp(ctx context.Context, cfg *appCfg) App { logSettings := &loggerSettings{} - log := pickLogger(cfg.config(), logSettings) + logLevel := newLogLevel(cfg.config()) + tagConfig := newTagsConfig(cfg.config(), logLevel.Level()) + logConfig := newLogLevelConfig(logLevel, tagConfig) + log := pickLogger(cfg.config(), logConfig.logLevel, logSettings, tagConfig) a := &app{ ctx: ctx, log: log.logger, - logLevel: log.lvl, cfg: cfg, loggerSettings: logSettings, webServer: new(fasthttp.Server), @@ -138,7 +225,7 @@ func newApp(ctx context.Context, cfg *appCfg) App { bucketCache: cache.NewBucketCache(getBucketCacheOptions(cfg.config(), log.logger), cfg.config().GetBool(cfgFeaturesTreePoolNetmapSupport)), } - a.initAppSettings() + a.initAppSettings(logConfig) // -- setup FastHTTP server -- a.webServer.Name = "frost-http-gw" @@ -172,11 +259,12 @@ func (a *app) config() *viper.Viper { return a.cfg.config() } -func (a *app) initAppSettings() { +func (a *app) initAppSettings(lc *logLevelConfig) { a.settings = &appSettings{ reconnectInterval: fetchReconnectInterval(a.config()), dialerSource: getDialerSource(a.log, a.config()), workerPoolSize: a.config().GetInt(cfgWorkerPoolSize), + logLevelConfig: lc, } a.settings.update(a.config(), a.log) } @@ -324,7 +412,7 @@ func (a *app) initResolver() { var err error a.resolver, err = resolver.NewContainerResolver(a.getResolverConfig()) if err != nil { - a.log.Fatal(logs.FailedToCreateResolver, zap.Error(err)) + a.log.Fatal(logs.FailedToCreateResolver, zap.Error(err), logs.TagField(logs.TagApp)) } } @@ -338,11 +426,12 @@ func (a *app) getResolverConfig() ([]string, *resolver.Config) { order := a.config().GetStringSlice(cfgResolveOrder) if resolveCfg.RPCAddress == "" { order = remove(order, resolver.NNSResolver) - a.log.Warn(logs.ResolverNNSWontBeUsedSinceRPCEndpointIsntProvided) + a.log.Warn(logs.ResolverNNSWontBeUsedSinceRPCEndpointIsntProvided, logs.TagField(logs.TagApp)) } if len(order) == 0 { - a.log.Info(logs.ContainerResolverWillBeDisabledBecauseOfResolversResolverOrderIsEmpty) + a.log.Info(logs.ContainerResolverWillBeDisabledBecauseOfResolversResolverOrderIsEmpty, + logs.TagField(logs.TagApp)) } return order, resolveCfg @@ -357,7 +446,7 @@ func (a *app) initMetrics() { func newGateMetrics(logger *zap.Logger, provider *metrics.GateMetrics, enabled bool) *gateMetrics { if !enabled { - logger.Warn(logs.MetricsAreDisabled) + logger.Warn(logs.MetricsAreDisabled, logs.TagField(logs.TagApp)) } return &gateMetrics{ logger: logger, @@ -375,7 +464,7 @@ func (m *gateMetrics) isEnabled() bool { func (m *gateMetrics) SetEnabled(enabled bool) { if !enabled { - m.logger.Warn(logs.MetricsAreDisabled) + m.logger.Warn(logs.MetricsAreDisabled, logs.TagField(logs.TagApp)) } m.mu.Lock() @@ -438,7 +527,7 @@ func getFrostFSKey(cfg *viper.Viper, log *zap.Logger) (*keys.PrivateKey, error) walletPath := cfg.GetString(cfgWalletPath) if len(walletPath) == 0 { - log.Info(logs.NoWalletPathSpecifiedCreatingEphemeralKeyAutomaticallyForThisRun) + log.Info(logs.NoWalletPathSpecifiedCreatingEphemeralKeyAutomaticallyForThisRun, logs.TagField(logs.TagApp)) key, err := keys.NewPrivateKey() if err != nil { return nil, err @@ -495,7 +584,10 @@ func getKeyFromWallet(w *wallet.Wallet, addrStr string, password *string) (*keys } func (a *app) Wait() { - a.log.Info(logs.StartingApplication, zap.String("app_name", "frostfs-http-gw"), zap.String("version", Version)) + a.log.Info(logs.StartingApplication, + zap.String("app_name", "frostfs-http-gw"), + zap.String("version", Version), + logs.TagField(logs.TagApp)) a.metrics.SetVersion(Version) a.setHealthStatus() @@ -526,10 +618,10 @@ func (a *app) Serve() { for i := range servs { go func(i int) { - a.log.Info(logs.StartingServer, zap.String("address", servs[i].Address())) + a.log.Info(logs.StartingServer, zap.String("address", servs[i].Address()), logs.TagField(logs.TagApp)) if err := a.webServer.Serve(servs[i].Listener()); err != nil && err != http.ErrServerClosed { a.metrics.MarkUnhealthy(servs[i].Address()) - a.log.Fatal(logs.ListenAndServe, zap.Error(err)) + a.log.Fatal(logs.ListenAndServe, zap.Error(err), logs.TagField(logs.TagApp)) } }(i) } @@ -551,7 +643,7 @@ LOOP: } } - a.log.Info(logs.ShuttingDownWebServer, zap.Error(a.webServer.Shutdown())) + a.log.Info(logs.ShuttingDownWebServer, zap.Error(a.webServer.Shutdown()), logs.TagField(logs.TagApp)) a.metrics.Shutdown() a.stopServices() @@ -561,7 +653,7 @@ LOOP: func (a *app) initWorkerPool() *ants.Pool { workerPool, err := ants.NewPool(a.settings.workerPoolSize) if err != nil { - a.log.Fatal(logs.FailedToCreateWorkerPool, zap.Error(err)) + a.log.Fatal(logs.FailedToCreateWorkerPool, zap.Error(err), logs.TagField(logs.TagApp)) } return workerPool } @@ -572,37 +664,33 @@ func (a *app) shutdownTracing() { defer cancel() if err := tracing.Shutdown(shdnCtx); err != nil { - a.log.Warn(logs.FailedToShutdownTracing, zap.Error(err)) + a.log.Warn(logs.FailedToShutdownTracing, zap.Error(err), logs.TagField(logs.TagApp)) } } func (a *app) configReload(ctx context.Context) { - a.log.Info(logs.SIGHUPConfigReloadStarted) + a.log.Info(logs.SIGHUPConfigReloadStarted, logs.TagField(logs.TagApp)) if !a.config().IsSet(cmdConfig) && !a.config().IsSet(cmdConfigDir) { - a.log.Warn(logs.FailedToReloadConfigBecauseItsMissed) + a.log.Warn(logs.FailedToReloadConfigBecauseItsMissed, logs.TagField(logs.TagApp)) return } if err := a.cfg.reload(); err != nil { - a.log.Warn(logs.FailedToReloadConfig, zap.Error(err)) + a.log.Warn(logs.FailedToReloadConfig, zap.Error(err), logs.TagField(logs.TagApp)) return } - if lvl, err := getLogLevel(a.config()); err != nil { - a.log.Warn(logs.LogLevelWontBeUpdated, zap.Error(err)) - } else { - a.logLevel.SetLevel(lvl) - } + a.settings.logLevelConfig.update(a.cfg.settings, a.log) if err := a.settings.dialerSource.Update(fetchMultinetConfig(a.config(), a.log)); err != nil { - a.log.Warn(logs.MultinetConfigWontBeUpdated, zap.Error(err)) + a.log.Warn(logs.MultinetConfigWontBeUpdated, zap.Error(err), logs.TagField(logs.TagApp)) } if err := a.resolver.UpdateResolvers(a.getResolverConfig()); err != nil { - a.log.Warn(logs.FailedToUpdateResolvers, zap.Error(err)) + a.log.Warn(logs.FailedToUpdateResolvers, zap.Error(err), logs.TagField(logs.TagApp)) } if err := a.updateServers(); err != nil { - a.log.Warn(logs.FailedToReloadServerParameters, zap.Error(err)) + a.log.Warn(logs.FailedToReloadServerParameters, zap.Error(err), logs.TagField(logs.TagApp)) } a.setRuntimeParameters() @@ -616,7 +704,7 @@ func (a *app) configReload(ctx context.Context) { a.initTracing(ctx) a.setHealthStatus() - a.log.Info(logs.SIGHUPConfigReloadCompleted) + a.log.Info(logs.SIGHUPConfigReloadCompleted, logs.TagField(logs.TagApp)) } func (a *app) startServices() { @@ -654,20 +742,20 @@ func (a *app) configureRouter(h *handler.Handler) { r.POST("/upload/{cid}", a.addMiddlewares(h.Upload)) r.OPTIONS("/upload/{cid}", a.addPreflight()) - a.log.Info(logs.AddedPathUploadCid) + a.log.Info(logs.AddedPathUploadCid, logs.TagField(logs.TagApp)) r.GET("/get/{cid}/{oid:*}", a.addMiddlewares(h.DownloadByAddressOrBucketName)) r.HEAD("/get/{cid}/{oid:*}", a.addMiddlewares(h.HeadByAddressOrBucketName)) r.OPTIONS("/get/{cid}/{oid:*}", a.addPreflight()) - a.log.Info(logs.AddedPathGetCidOid) + a.log.Info(logs.AddedPathGetCidOid, logs.TagField(logs.TagApp)) r.GET("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.addMiddlewares(h.DownloadByAttribute)) r.HEAD("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.addMiddlewares(h.HeadByAttribute)) r.OPTIONS("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.addPreflight()) - a.log.Info(logs.AddedPathGetByAttributeCidAttrKeyAttrVal) + a.log.Info(logs.AddedPathGetByAttributeCidAttrKeyAttrVal, logs.TagField(logs.TagApp)) r.GET("/zip/{cid}/{prefix:*}", a.addMiddlewares(h.DownloadZip)) r.OPTIONS("/zip/{cid}/{prefix:*}", a.addPreflight()) r.GET("/tar/{cid}/{prefix:*}", a.addMiddlewares(h.DownloadTar)) r.OPTIONS("/tar/{cid}/{prefix:*}", a.addPreflight()) - a.log.Info(logs.AddedPathZipCidPrefix) + a.log.Info(logs.AddedPathZipCidPrefix, logs.TagField(logs.TagApp)) a.webServer.Handler = r.Handler } @@ -756,14 +844,11 @@ func (a *app) logger(h fasthttp.RequestHandler) fasthttp.RequestHandler { reqCtx = utils.SetReqLog(reqCtx, log) utils.SetContextToRequest(reqCtx, req) - fields := []zap.Field{ - zap.String("remote", req.RemoteAddr().String()), + log.Info(logs.Request, zap.String("remote", req.RemoteAddr().String()), zap.ByteString("method", req.Method()), zap.ByteString("path", req.Path()), zap.ByteString("query", req.QueryArgs().QueryString()), - } - - log.Info(logs.Request, fields...) + logs.TagField(logs.TagDatapath)) h(req) } } @@ -807,7 +892,7 @@ func (a *app) tokenizer(h fasthttp.RequestHandler) fasthttp.RequestHandler { if err != nil { log := utils.GetReqLogOrDefault(reqCtx, a.log) - log.Error(logs.CouldNotFetchAndStoreBearerToken, zap.Error(err)) + log.Error(logs.CouldNotFetchAndStoreBearerToken, zap.Error(err), logs.TagField(logs.TagDatapath)) handler.ResponseError(req, "could not fetch and store bearer token: "+err.Error(), fasthttp.StatusBadRequest) return } @@ -866,17 +951,17 @@ func (a *app) initServers(ctx context.Context) { if err != nil { a.unbindServers = append(a.unbindServers, serverInfo) a.metrics.MarkUnhealthy(serverInfo.Address) - a.log.Warn(logs.FailedToAddServer, append(fields, zap.Error(err))...) + a.log.Warn(logs.FailedToAddServer, append(fields, zap.Error(err), logs.TagField(logs.TagApp))...) continue } a.metrics.MarkHealthy(serverInfo.Address) a.servers = append(a.servers, srv) - a.log.Info(logs.AddServer, fields...) + a.log.Info(logs.AddServer, append(fields, logs.TagField(logs.TagApp))...) } if len(a.servers) == 0 { - a.log.Fatal(logs.NoHealthyServers) + a.log.Fatal(logs.NoHealthyServers, logs.TagField(logs.TagApp)) } } @@ -950,13 +1035,14 @@ func (a *app) initTracing(ctx context.Context) { if trustedCa := a.config().GetString(cfgTracingTrustedCa); trustedCa != "" { caBytes, err := os.ReadFile(trustedCa) if err != nil { - a.log.Warn(logs.FailedToInitializeTracing, zap.Error(err)) + a.log.Warn(logs.FailedToInitializeTracing, zap.Error(err), logs.TagField(logs.TagApp)) return } certPool := x509.NewCertPool() ok := certPool.AppendCertsFromPEM(caBytes) if !ok { - a.log.Warn(logs.FailedToInitializeTracing, zap.String("error", "can't fill cert pool by ca cert")) + a.log.Warn(logs.FailedToInitializeTracing, zap.String("error", "can't fill cert pool by ca cert"), + logs.TagField(logs.TagApp)) return } cfg.ServerCaCertPool = certPool @@ -964,24 +1050,24 @@ func (a *app) initTracing(ctx context.Context) { attributes, err := fetchTracingAttributes(a.config()) if err != nil { - a.log.Warn(logs.FailedToInitializeTracing, zap.Error(err)) + a.log.Warn(logs.FailedToInitializeTracing, zap.Error(err), logs.TagField(logs.TagApp)) return } cfg.Attributes = attributes updated, err := tracing.Setup(ctx, cfg) if err != nil { - a.log.Warn(logs.FailedToInitializeTracing, zap.Error(err)) + a.log.Warn(logs.FailedToInitializeTracing, zap.Error(err), logs.TagField(logs.TagApp)) } if updated { - a.log.Info(logs.TracingConfigUpdated) + a.log.Info(logs.TracingConfigUpdated, logs.TagField(logs.TagApp)) } } func (a *app) setRuntimeParameters() { if len(os.Getenv("GOMEMLIMIT")) != 0 { // default limit < yaml limit < app env limit < GOMEMLIMIT - a.log.Warn(logs.RuntimeSoftMemoryDefinedWithGOMEMLIMIT) + a.log.Warn(logs.RuntimeSoftMemoryDefinedWithGOMEMLIMIT, logs.TagField(logs.TagApp)) return } @@ -990,7 +1076,8 @@ func (a *app) setRuntimeParameters() { if softMemoryLimit != previous { a.log.Info(logs.RuntimeSoftMemoryLimitUpdated, zap.Int64("new_value", softMemoryLimit), - zap.Int64("old_value", previous)) + zap.Int64("old_value", previous), + logs.TagField(logs.TagApp)) } } @@ -1016,34 +1103,32 @@ func (a *app) tryReconnect(ctx context.Context, sr *fasthttp.Server) bool { a.mu.Lock() defer a.mu.Unlock() - a.log.Info(logs.ServerReconnecting) + a.log.Info(logs.ServerReconnecting, logs.TagField(logs.TagApp)) var failedServers []ServerInfo for _, serverInfo := range a.unbindServers { - fields := []zap.Field{ - zap.String("address", serverInfo.Address), zap.Bool("tls enabled", serverInfo.TLS.Enabled), - zap.String("tls cert", serverInfo.TLS.CertFile), zap.String("tls key", serverInfo.TLS.KeyFile), - } - srv, err := newServer(ctx, serverInfo) if err != nil { - a.log.Warn(logs.ServerReconnectFailed, zap.Error(err)) + a.log.Warn(logs.ServerReconnectFailed, zap.Error(err), logs.TagField(logs.TagApp)) failedServers = append(failedServers, serverInfo) a.metrics.MarkUnhealthy(serverInfo.Address) continue } go func() { - a.log.Info(logs.StartingServer, zap.String("address", srv.Address())) + a.log.Info(logs.StartingServer, zap.String("address", srv.Address()), logs.TagField(logs.TagApp)) a.metrics.MarkHealthy(serverInfo.Address) if err = sr.Serve(srv.Listener()); err != nil && !errors.Is(err, http.ErrServerClosed) { - a.log.Warn(logs.ListenAndServe, zap.Error(err)) + a.log.Warn(logs.ListenAndServe, zap.Error(err), logs.TagField(logs.TagApp)) a.metrics.MarkUnhealthy(serverInfo.Address) } }() a.servers = append(a.servers, srv) - a.log.Info(logs.ServerReconnectedSuccessfully, fields...) + a.log.Info(logs.ServerReconnectedSuccessfully, + zap.String("address", serverInfo.Address), zap.Bool("tls enabled", serverInfo.TLS.Enabled), + zap.String("tls cert", serverInfo.TLS.CertFile), zap.String("tls key", serverInfo.TLS.KeyFile), + logs.TagField(logs.TagApp)) } a.unbindServers = failedServers diff --git a/cmd/http-gw/logger.go b/cmd/http-gw/logger.go new file mode 100644 index 0000000..91105f7 --- /dev/null +++ b/cmd/http-gw/logger.go @@ -0,0 +1,174 @@ +package main + +import ( + "fmt" + "os" + + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" + "git.frostfs.info/TrueCloudLab/zapjournald" + "github.com/spf13/viper" + "github.com/ssgreg/journald" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +func getLogLevel(v *viper.Viper) (zapcore.Level, error) { + var lvl zapcore.Level + lvlStr := v.GetString(cfgLoggerLevel) + err := lvl.UnmarshalText([]byte(lvlStr)) + if err != nil { + return lvl, fmt.Errorf("incorrect logger level configuration %s (%v), "+ + "value should be one of %v", lvlStr, err, [...]zapcore.Level{ + zapcore.DebugLevel, + zapcore.InfoLevel, + zapcore.WarnLevel, + zapcore.ErrorLevel, + zapcore.DPanicLevel, + zapcore.PanicLevel, + zapcore.FatalLevel, + }) + } + return lvl, nil +} + +var _ zapcore.Core = (*zapCoreTagFilterWrapper)(nil) + +type zapCoreTagFilterWrapper struct { + core zapcore.Core + settings TagFilterSettings + extra []zap.Field +} + +type TagFilterSettings interface { + LevelEnabled(tag string, lvl zapcore.Level) bool +} + +func (c *zapCoreTagFilterWrapper) Enabled(level zapcore.Level) bool { + return c.core.Enabled(level) +} + +func (c *zapCoreTagFilterWrapper) With(fields []zapcore.Field) zapcore.Core { + return &zapCoreTagFilterWrapper{ + core: c.core.With(fields), + settings: c.settings, + extra: append(c.extra, fields...), + } +} + +func (c *zapCoreTagFilterWrapper) Check(entry zapcore.Entry, checked *zapcore.CheckedEntry) *zapcore.CheckedEntry { + if c.core.Enabled(entry.Level) { + return checked.AddCore(entry, c) + } + return checked +} + +func (c *zapCoreTagFilterWrapper) Write(entry zapcore.Entry, fields []zapcore.Field) error { + if c.shouldSkip(entry, fields) || c.shouldSkip(entry, c.extra) { + return nil + } + + return c.core.Write(entry, fields) +} + +func (c *zapCoreTagFilterWrapper) shouldSkip(entry zapcore.Entry, fields []zap.Field) bool { + for _, field := range fields { + if field.Key == logs.TagFieldName && field.Type == zapcore.StringType { + if !c.settings.LevelEnabled(field.String, entry.Level) { + return true + } + break + } + } + + return false +} + +func (c *zapCoreTagFilterWrapper) Sync() error { + return c.core.Sync() +} + +func applyZapCoreMiddlewares(core zapcore.Core, v *viper.Viper, loggerSettings LoggerAppSettings, tagSetting TagFilterSettings) zapcore.Core { + core = &zapCoreTagFilterWrapper{ + core: core, + settings: tagSetting, + } + + if v.GetBool(cfgLoggerSamplingEnabled) { + core = zapcore.NewSamplerWithOptions(core, + v.GetDuration(cfgLoggerSamplingInterval), + v.GetInt(cfgLoggerSamplingInitial), + v.GetInt(cfgLoggerSamplingThereafter), + zapcore.SamplerHook(func(_ zapcore.Entry, dec zapcore.SamplingDecision) { + if dec&zapcore.LogDropped > 0 { + loggerSettings.DroppedLogsInc() + } + })) + } + + return core +} + +func newLogEncoder() zapcore.Encoder { + c := zap.NewProductionEncoderConfig() + c.EncodeTime = zapcore.ISO8601TimeEncoder + + return zapcore.NewConsoleEncoder(c) +} + +// newStdoutLogger constructs a zap.Logger instance for current application. +// Panics on failure. +// +// Logger is built from zap's production logging configuration with: +// - parameterized level (debug by default) +// - console encoding +// - ISO8601 time encoding +// +// Logger records a stack trace for all messages at or above fatal level. +// +// See also zapcore.Level, zap.NewProductionConfig, zap.AddStacktrace. +func newStdoutLogger(v *viper.Viper, lvl zap.AtomicLevel, loggerSettings LoggerAppSettings, tagSetting TagFilterSettings) *Logger { + stdout := zapcore.AddSync(os.Stderr) + + consoleOutCore := zapcore.NewCore(newLogEncoder(), stdout, lvl) + consoleOutCore = applyZapCoreMiddlewares(consoleOutCore, v, loggerSettings, tagSetting) + + return &Logger{ + logger: zap.New(consoleOutCore, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel))), + lvl: lvl, + } +} + +func newJournaldLogger(v *viper.Viper, lvl zap.AtomicLevel, loggerSettings LoggerAppSettings, tagSetting TagFilterSettings) *Logger { + encoder := zapjournald.NewPartialEncoder(newLogEncoder(), zapjournald.SyslogFields) + + core := zapjournald.NewCore(lvl, encoder, &journald.Journal{}, zapjournald.SyslogFields) + coreWithContext := core.With([]zapcore.Field{ + zapjournald.SyslogFacility(zapjournald.LogDaemon), + zapjournald.SyslogIdentifier(), + zapjournald.SyslogPid(), + }) + + coreWithContext = applyZapCoreMiddlewares(coreWithContext, v, loggerSettings, tagSetting) + + return &Logger{ + logger: zap.New(coreWithContext, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel))), + lvl: lvl, + } +} + +type LoggerAppSettings interface { + DroppedLogsInc() +} + +func pickLogger(v *viper.Viper, lvl zap.AtomicLevel, loggerSettings LoggerAppSettings, tagSettings TagFilterSettings) *Logger { + dest := v.GetString(cfgLoggerDestination) + + switch dest { + case destinationStdout: + return newStdoutLogger(v, lvl, loggerSettings, tagSettings) + case destinationJournald: + return newJournaldLogger(v, lvl, loggerSettings, tagSettings) + default: + panic(fmt.Sprintf("wrong destination for logger: %s", dest)) + } +} diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index 5670f87..d8331ad 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -23,10 +23,8 @@ import ( grpctracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" treepool "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree" - "git.frostfs.info/TrueCloudLab/zapjournald" "github.com/spf13/pflag" "github.com/spf13/viper" - "github.com/ssgreg/journald" "github.com/valyala/fasthttp" "go.uber.org/zap" "go.uber.org/zap/zapcore" @@ -111,6 +109,11 @@ const ( cfgLoggerSamplingThereafter = "logger.sampling.thereafter" cfgLoggerSamplingInterval = "logger.sampling.interval" + cfgLoggerTags = "logger.tags" + cfgLoggerTagsPrefixTmpl = cfgLoggerTags + ".%d." + cfgLoggerTagsNameTmpl = cfgLoggerTagsPrefixTmpl + "name" + cfgLoggerTagsLevelTmpl = cfgLoggerTagsPrefixTmpl + "level" + // Wallet. cfgWalletPassphrase = "wallet.passphrase" cfgWalletPath = "wallet.path" @@ -193,6 +196,8 @@ var ignore = map[string]struct{}{ cmdVersion: {}, } +var defaultTags = []string{logs.TagApp, logs.TagDatapath} + type Logger struct { logger *zap.Logger lvl zap.AtomicLevel @@ -499,112 +504,33 @@ func mergeConfig(v *viper.Viper, fileName string) error { return v.MergeConfig(cfgFile) } -type LoggerAppSettings interface { - DroppedLogsInc() -} +func fetchLogTagsConfig(v *viper.Viper, defaultLvl zapcore.Level) (map[string]zapcore.Level, error) { + res := make(map[string]zapcore.Level) -func pickLogger(v *viper.Viper, settings LoggerAppSettings) *Logger { - lvl, err := getLogLevel(v) - if err != nil { - panic(err) + for i := 0; ; i++ { + name := v.GetString(fmt.Sprintf(cfgLoggerTagsNameTmpl, i)) + if name == "" { + break + } + + lvl := defaultLvl + level := v.GetString(fmt.Sprintf(cfgLoggerTagsLevelTmpl, i)) + if level != "" { + if err := lvl.Set(level); err != nil { + return nil, fmt.Errorf("failed to parse log tags config, unknown level: '%s'", level) + } + } + + res[name] = lvl } - dest := v.GetString(cfgLoggerDestination) - - switch dest { - case destinationStdout: - return newStdoutLogger(v, lvl, settings) - case destinationJournald: - return newJournaldLogger(v, lvl, settings) - default: - panic(fmt.Sprintf("wrong destination for logger: %s", dest)) - } -} - -// newStdoutLogger constructs a zap.Logger instance for current application. -// Panics on failure. -// -// Logger is built from zap's production logging configuration with: -// - parameterized level (debug by default) -// - console encoding -// - ISO8601 time encoding -// -// Logger records a stack trace for all messages at or above fatal level. -// -// See also zapcore.Level, zap.NewProductionConfig, zap.AddStacktrace. -func newStdoutLogger(v *viper.Viper, lvl zapcore.Level, settings LoggerAppSettings) *Logger { - stdout := zapcore.AddSync(os.Stderr) - level := zap.NewAtomicLevelAt(lvl) - - consoleOutCore := zapcore.NewCore(newLogEncoder(), stdout, level) - consoleOutCore = applyZapCoreMiddlewares(consoleOutCore, v, settings) - - return &Logger{ - logger: zap.New(consoleOutCore, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel))), - lvl: level, - } -} - -func newJournaldLogger(v *viper.Viper, lvl zapcore.Level, settings LoggerAppSettings) *Logger { - level := zap.NewAtomicLevelAt(lvl) - - encoder := zapjournald.NewPartialEncoder(newLogEncoder(), zapjournald.SyslogFields) - - core := zapjournald.NewCore(level, encoder, &journald.Journal{}, zapjournald.SyslogFields) - coreWithContext := core.With([]zapcore.Field{ - zapjournald.SyslogFacility(zapjournald.LogDaemon), - zapjournald.SyslogIdentifier(), - zapjournald.SyslogPid(), - }) - - coreWithContext = applyZapCoreMiddlewares(coreWithContext, v, settings) - - return &Logger{ - logger: zap.New(coreWithContext, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel))), - lvl: level, - } -} - -func newLogEncoder() zapcore.Encoder { - c := zap.NewProductionEncoderConfig() - c.EncodeTime = zapcore.ISO8601TimeEncoder - - return zapcore.NewConsoleEncoder(c) -} - -func applyZapCoreMiddlewares(core zapcore.Core, v *viper.Viper, settings LoggerAppSettings) zapcore.Core { - if v.GetBool(cfgLoggerSamplingEnabled) { - core = zapcore.NewSamplerWithOptions(core, - v.GetDuration(cfgLoggerSamplingInterval), - v.GetInt(cfgLoggerSamplingInitial), - v.GetInt(cfgLoggerSamplingThereafter), - zapcore.SamplerHook(func(_ zapcore.Entry, dec zapcore.SamplingDecision) { - if dec&zapcore.LogDropped > 0 { - settings.DroppedLogsInc() - } - })) + if len(res) == 0 && !v.IsSet(cfgLoggerTags) { + for _, tag := range defaultTags { + res[tag] = defaultLvl + } } - return core -} - -func getLogLevel(v *viper.Viper) (zapcore.Level, error) { - var lvl zapcore.Level - lvlStr := v.GetString(cfgLoggerLevel) - err := lvl.UnmarshalText([]byte(lvlStr)) - if err != nil { - return lvl, fmt.Errorf("incorrect logger level configuration %s (%v), "+ - "value should be one of %v", lvlStr, err, [...]zapcore.Level{ - zapcore.DebugLevel, - zapcore.InfoLevel, - zapcore.WarnLevel, - zapcore.ErrorLevel, - zapcore.DPanicLevel, - zapcore.PanicLevel, - zapcore.FatalLevel, - }) - } - return lvl, nil + return res, nil } func fetchReconnectInterval(cfg *viper.Viper) time.Duration { @@ -620,20 +546,19 @@ func fetchIndexPageTemplate(v *viper.Viper, l *zap.Logger) (string, bool) { if !v.GetBool(cfgIndexPageEnabled) { return "", false } - reader, err := os.Open(v.GetString(cfgIndexPageTemplatePath)) if err != nil { - l.Warn(logs.FailedToReadIndexPageTemplate, zap.Error(err)) + l.Warn(logs.FailedToReadIndexPageTemplate, zap.Error(err), logs.TagField(logs.TagApp)) return "", true } tmpl, err := io.ReadAll(reader) if err != nil { - l.Warn(logs.FailedToReadIndexPageTemplate, zap.Error(err)) + l.Warn(logs.FailedToReadIndexPageTemplate, zap.Error(err), logs.TagField(logs.TagApp)) return "", true } - l.Info(logs.SetCustomIndexPageTemplate) + l.Info(logs.SetCustomIndexPageTemplate, logs.TagField(logs.TagApp)) return string(tmpl), true } @@ -674,7 +599,7 @@ func fetchServers(v *viper.Viper, log *zap.Logger) []ServerInfo { } if _, ok := seen[serverInfo.Address]; ok { - log.Warn(logs.WarnDuplicateAddress, zap.String("address", serverInfo.Address)) + log.Warn(logs.WarnDuplicateAddress, zap.String("address", serverInfo.Address), logs.TagField(logs.TagApp)) continue } seen[serverInfo.Address] = struct{}{} @@ -687,7 +612,7 @@ func fetchServers(v *viper.Viper, log *zap.Logger) []ServerInfo { func (a *app) initPools(ctx context.Context) { key, err := getFrostFSKey(a.config(), a.log) if err != nil { - a.log.Fatal(logs.CouldNotLoadFrostFSPrivateKey, zap.Error(err)) + a.log.Fatal(logs.CouldNotLoadFrostFSPrivateKey, zap.Error(err), logs.TagField(logs.TagApp)) } var prm pool.InitParameters @@ -695,7 +620,8 @@ func (a *app) initPools(ctx context.Context) { prm.SetKey(&key.PrivateKey) prmTree.SetKey(key) - a.log.Info(logs.UsingCredentials, zap.String("FrostFS", hex.EncodeToString(key.PublicKey().Bytes()))) + a.log.Info(logs.UsingCredentials, zap.String("FrostFS", hex.EncodeToString(key.PublicKey().Bytes())), + logs.TagField(logs.TagApp)) for _, peer := range fetchPeers(a.log, a.config()) { prm.AddNode(peer) @@ -750,11 +676,11 @@ func (a *app) initPools(ctx context.Context) { p, err := pool.NewPool(prm) if err != nil { - a.log.Fatal(logs.FailedToCreateConnectionPool, zap.Error(err)) + a.log.Fatal(logs.FailedToCreateConnectionPool, zap.Error(err), logs.TagField(logs.TagApp)) } if err = p.Dial(ctx); err != nil { - a.log.Fatal(logs.FailedToDialConnectionPool, zap.Error(err)) + a.log.Fatal(logs.FailedToDialConnectionPool, zap.Error(err), logs.TagField(logs.TagApp)) } if a.config().GetBool(cfgFeaturesTreePoolNetmapSupport) { @@ -763,10 +689,10 @@ func (a *app) initPools(ctx context.Context) { treePool, err := treepool.NewPool(prmTree) if err != nil { - a.log.Fatal(logs.FailedToCreateTreePool, zap.Error(err)) + a.log.Fatal(logs.FailedToCreateTreePool, zap.Error(err), logs.TagField(logs.TagApp)) } if err = treePool.Dial(ctx); err != nil { - a.log.Fatal(logs.FailedToDialTreePool, zap.Error(err)) + a.log.Fatal(logs.FailedToDialTreePool, zap.Error(err), logs.TagField(logs.TagApp)) } a.pool = p @@ -797,7 +723,8 @@ func fetchPeers(l *zap.Logger, v *viper.Viper) []pool.NodeParam { l.Info(logs.AddedStoragePeer, zap.Int("priority", priority), zap.String("address", address), - zap.Float64("weight", weight)) + zap.Float64("weight", weight), + logs.TagField(logs.TagApp)) } return nodes @@ -836,7 +763,8 @@ func fetchCacheLifetime(v *viper.Viper, l *zap.Logger, cfgEntry string, defaultV l.Error(logs.InvalidLifetimeUsingDefaultValue, zap.String("parameter", cfgEntry), zap.Duration("value in config", lifetime), - zap.Duration("default", defaultValue)) + zap.Duration("default", defaultValue), + logs.TagField(logs.TagApp)) } else { return lifetime } @@ -852,7 +780,8 @@ func fetchCacheSize(v *viper.Viper, l *zap.Logger, cfgEntry string, defaultValue l.Error(logs.InvalidCacheSizeUsingDefaultValue, zap.String("parameter", cfgEntry), zap.Int("value in config", size), - zap.Int("default", defaultValue)) + zap.Int("default", defaultValue), + logs.TagField(logs.TagApp)) } else { return size } @@ -864,7 +793,7 @@ func fetchCacheSize(v *viper.Viper, l *zap.Logger, cfgEntry string, defaultValue func getDialerSource(logger *zap.Logger, cfg *viper.Viper) *internalnet.DialerSource { source, err := internalnet.NewDialerSource(fetchMultinetConfig(cfg, logger)) if err != nil { - logger.Fatal(logs.FailedToLoadMultinetConfig, zap.Error(err)) + logger.Fatal(logs.FailedToLoadMultinetConfig, zap.Error(err), logs.TagField(logs.TagApp)) } return source } diff --git a/config/config.env b/config/config.env index db619b5..af0eba1 100644 --- a/config/config.env +++ b/config/config.env @@ -20,6 +20,8 @@ HTTP_GW_LOGGER_SAMPLING_ENABLED=false HTTP_GW_LOGGER_SAMPLING_INITIAL=100 HTTP_GW_LOGGER_SAMPLING_THEREAFTER=100 HTTP_GW_LOGGER_SAMPLING_INTERVAL=1s +HTTP_GW_LOGGER_TAGS_0_NAME=app +HTTP_GW_LOGGER_TAGS_1_NAME=datapath HTTP_GW_SERVER_0_ADDRESS=0.0.0.0:443 HTTP_GW_SERVER_0_TLS_ENABLED=false diff --git a/config/config.yaml b/config/config.yaml index a70ec9a..8c51591 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -29,6 +29,10 @@ logger: initial: 100 thereafter: 100 interval: 1s + tags: + - name: app + - name: datapath + level: debug server: - address: 0.0.0.0:8080 diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index 6aadd1f..4f9bc3b 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -174,6 +174,11 @@ logger: initial: 100 thereafter: 100 interval: 1s + tags: + - name: "app" + level: info + - name: "datapath" + - name: "external_storage_tree" ``` | Parameter | Type | SIGHUP reload | Default value | Description | @@ -184,6 +189,30 @@ logger: | `sampling.initial` | `int` | no | '100' | Sampling count of first log entries. | | `sampling.thereafter` | `int` | no | '100' | Sampling count of entries after an `interval`. | | `sampling.interval` | `duration` | no | '1s' | Sampling interval of messaging similar entries. | +| `sampling.tags` | `[]Tag` | yes | | Tagged log entries that should be additionally logged (available tags see in the next section). | + +## Tags + +There are additional log entries that can hurt performance and can be additionally logged by using `logger.tags` +parameter. Available tags: + +```yaml +tags: + - name: "app" + level: info +``` + +| Parameter | Type | SIGHUP reload | Default value | Description | +|-----------------------|------------|---------------|---------------------------|-------------------------------------------------------------------------------------------------------| +| `name` | `string` | yes | | Tag name. Possible values see below in `Tag values` section. | +| `level` | `string` | yes | Value from `logger.level` | Logging level for specific tag. Possible values: `debug`, `info`, `warn`, `dpanic`, `panic`, `fatal`. | + +### Tag values + +* `app` - common application logs (enabled by default). +* `datapath` - main logic of application (enabled by default). +* `external_storage` - external interaction with storage node. +* `external_storage_tree` - external interaction with tree service in storage node. # `web` section diff --git a/internal/cache/buckets.go b/internal/cache/buckets.go index 2fa8f25..91ae5b2 100644 --- a/internal/cache/buckets.go +++ b/internal/cache/buckets.go @@ -72,7 +72,7 @@ func (o *BucketCache) GetByCID(cnrID cid.ID) *data.BucketInfo { key, ok := entry.(string) if !ok { o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)), - zap.String("expected", fmt.Sprintf("%T", key))) + zap.String("expected", fmt.Sprintf("%T", key)), logs.TagField(logs.TagDatapath)) return nil } @@ -88,7 +88,7 @@ func (o *BucketCache) get(key string) *data.BucketInfo { result, ok := entry.(*data.BucketInfo) if !ok { o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)), - zap.String("expected", fmt.Sprintf("%T", result))) + zap.String("expected", fmt.Sprintf("%T", result)), logs.TagField(logs.TagDatapath)) return nil } diff --git a/internal/cache/netmap.go b/internal/cache/netmap.go index 6d91fe7..ce01b47 100644 --- a/internal/cache/netmap.go +++ b/internal/cache/netmap.go @@ -53,7 +53,7 @@ func (c *NetmapCache) Get() *netmap.NetMap { result, ok := entry.(netmap.NetMap) if !ok { c.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)), - zap.String("expected", fmt.Sprintf("%T", result))) + zap.String("expected", fmt.Sprintf("%T", result)), logs.TagField(logs.TagDatapath)) return nil } diff --git a/internal/handler/browse.go b/internal/handler/browse.go index 64ad1f5..2d0e34d 100644 --- a/internal/handler/browse.go +++ b/internal/handler/browse.go @@ -230,7 +230,7 @@ func (h *Handler) getDirObjectsNative(ctx context.Context, bucketInfo *data.Buck } for objExt := range resp { if objExt.Error != nil { - log.Error(logs.FailedToHeadObject, zap.Error(objExt.Error)) + log.Error(logs.FailedToHeadObject, zap.Error(objExt.Error), logs.TagField(logs.TagExternalStorage)) result.hasErrors = true continue } @@ -273,7 +273,7 @@ func (h *Handler) headDirObjects(ctx context.Context, cnrID cid.ID, objectIDs Re }) if err != nil { wg.Done() - log.Warn(logs.FailedToSumbitTaskToPool, zap.Error(err)) + log.Warn(logs.FailedToSumbitTaskToPool, zap.Error(err), logs.TagField(logs.TagDatapath)) } select { case <-ctx.Done(): @@ -283,7 +283,7 @@ func (h *Handler) headDirObjects(ctx context.Context, cnrID cid.ID, objectIDs Re } }) if err != nil { - log.Error(logs.FailedToIterateOverResponse, zap.Error(err)) + log.Error(logs.FailedToIterateOverResponse, zap.Error(err), logs.TagField(logs.TagDatapath)) } wg.Wait() }() diff --git a/internal/handler/download.go b/internal/handler/download.go index d5fac23..783fe09 100644 --- a/internal/handler/download.go +++ b/internal/handler/download.go @@ -43,6 +43,8 @@ func (h *Handler) DownloadByAddressOrBucketName(c *fasthttp.RequestCtx) { checkS3Err := h.tree.CheckSettingsNodeExists(ctx, bktInfo) if checkS3Err != nil && !errors.Is(checkS3Err, layer.ErrNodeNotFound) { + log.Error(logs.FailedToCheckIfSettingsNodeExist, zap.String("cid", bktInfo.CID.String()), + zap.Error(checkS3Err), logs.TagField(logs.TagExternalStorageTree)) logAndSendBucketError(c, log, checkS3Err) return } @@ -121,13 +123,13 @@ func (h *Handler) getZipResponseWriter(ctx context.Context, log *zap.Logger, res }), ) if errIter != nil { - log.Error(logs.IteratingOverSelectedObjectsFailed, zap.Error(errIter)) + log.Error(logs.IteratingOverSelectedObjectsFailed, zap.Error(errIter), logs.TagField(logs.TagDatapath)) return } else if objectsWritten == 0 { - log.Warn(logs.ObjectsNotFound) + log.Warn(logs.ObjectsNotFound, logs.TagField(logs.TagDatapath)) } if err := zipWriter.Close(); err != nil { - log.Error(logs.CloseZipWriter, zap.Error(err)) + log.Error(logs.CloseZipWriter, zap.Error(err), logs.TagField(logs.TagDatapath)) } } } @@ -187,10 +189,10 @@ func (h *Handler) getTarResponseWriter(ctx context.Context, log *zap.Logger, res defer func() { if err := tarWriter.Close(); err != nil { - log.Error(logs.CloseTarWriter, zap.Error(err)) + log.Error(logs.CloseTarWriter, zap.Error(err), logs.TagField(logs.TagDatapath)) } if err := gzipWriter.Close(); err != nil { - log.Error(logs.CloseGzipWriter, zap.Error(err)) + log.Error(logs.CloseGzipWriter, zap.Error(err), logs.TagField(logs.TagDatapath)) } }() @@ -204,9 +206,9 @@ func (h *Handler) getTarResponseWriter(ctx context.Context, log *zap.Logger, res }), ) if errIter != nil { - log.Error(logs.IteratingOverSelectedObjectsFailed, zap.Error(errIter)) + log.Error(logs.IteratingOverSelectedObjectsFailed, zap.Error(errIter), logs.TagField(logs.TagDatapath)) } else if objectsWritten == 0 { - log.Warn(logs.ObjectsNotFound) + log.Warn(logs.ObjectsNotFound, logs.TagField(logs.TagDatapath)) } } } @@ -237,18 +239,18 @@ func (h *Handler) putObjectToArchive(ctx context.Context, log *zap.Logger, cnrID resGet, err := h.frostfs.GetObject(ctx, prm) if err != nil { - log.Error(logs.FailedToGetObject, zap.Error(err)) + log.Error(logs.FailedToGetObject, zap.Error(err), logs.TagField(logs.TagExternalStorage)) return false } fileWriter, err := createArchiveHeader(&resGet.Header) if err != nil { - log.Error(logs.FailedToAddObjectToArchive, zap.Error(err)) + log.Error(logs.FailedToAddObjectToArchive, zap.Error(err), logs.TagField(logs.TagDatapath)) return false } if err = writeToArchive(resGet, fileWriter, buf); err != nil { - log.Error(logs.FailedToAddObjectToArchive, zap.Error(err)) + log.Error(logs.FailedToAddObjectToArchive, zap.Error(err), logs.TagField(logs.TagDatapath)) return false } @@ -264,7 +266,8 @@ func (h *Handler) searchObjectsByPrefix(c *fasthttp.RequestCtx, log *zap.Logger, prefix, err := url.QueryUnescape(prefix) if err != nil { - log.Error(logs.FailedToUnescapeQuery, zap.String("cid", scid), zap.String("prefix", prefix), zap.Error(err)) + log.Error(logs.FailedToUnescapeQuery, zap.String("cid", scid), zap.String("prefix", prefix), + zap.Error(err), logs.TagField(logs.TagDatapath)) ResponseError(c, "could not unescape prefix: "+err.Error(), fasthttp.StatusBadRequest) return nil, err } @@ -273,7 +276,7 @@ func (h *Handler) searchObjectsByPrefix(c *fasthttp.RequestCtx, log *zap.Logger, resSearch, err := h.search(ctx, cnrID, object.AttributeFilePath, prefix, object.MatchCommonPrefix) if err != nil { - log.Error(logs.CouldNotSearchForObjects, zap.Error(err)) + log.Error(logs.CouldNotSearchForObjects, zap.Error(err), logs.TagField(logs.TagExternalStorage)) ResponseError(c, "could not search for objects: "+err.Error(), fasthttp.StatusBadRequest) return nil, err } diff --git a/internal/handler/filter.go b/internal/handler/filter.go index 745718a..da99db7 100644 --- a/internal/handler/filter.go +++ b/internal/handler/filter.go @@ -50,7 +50,8 @@ func filterHeaders(l *zap.Logger, header *fasthttp.RequestHeader) (map[string]st l.Debug(logs.AddAttributeToResultObject, zap.String("key", k), - zap.String("val", v)) + zap.String("val", v), + logs.TagField(logs.TagDatapath)) }) return result, err diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 4e23c3b..b27c607 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -206,11 +206,13 @@ func (h *Handler) byS3Path(ctx context.Context, req request, cnrID cid.ID, path foundOID, err := h.tree.GetLatestVersion(ctx, &cnrID, path) if err != nil { + log.Error(logs.FailedToGetLatestVersionOfObject, zap.Error(err), zap.String("cid", cnrID.String()), + zap.String("path", path), logs.TagField(logs.TagExternalStorageTree)) logAndSendBucketError(c, log, err) return } if foundOID.IsDeleteMarker { - log.Error(logs.ObjectWasDeleted) + log.Error(logs.ObjectWasDeleted, logs.TagField(logs.TagExternalStorageTree)) ResponseError(c, "object deleted", fasthttp.StatusNotFound) return } @@ -230,14 +232,16 @@ func (h *Handler) byAttribute(c *fasthttp.RequestCtx, handler func(context.Conte key, err := url.QueryUnescape(key) if err != nil { - log.Error(logs.FailedToUnescapeQuery, zap.String("cid", cidParam), zap.String("attr_key", key), zap.Error(err)) + log.Error(logs.FailedToUnescapeQuery, zap.String("cid", cidParam), zap.String("attr_key", key), + zap.Error(err), logs.TagField(logs.TagDatapath)) ResponseError(c, "could not unescape attr_key: "+err.Error(), fasthttp.StatusBadRequest) return } val, err = url.QueryUnescape(val) if err != nil { - log.Error(logs.FailedToUnescapeQuery, zap.String("cid", cidParam), zap.String("attr_val", val), zap.Error(err)) + log.Error(logs.FailedToUnescapeQuery, zap.String("cid", cidParam), zap.String("attr_val", val), + zap.Error(err), logs.TagField(logs.TagDatapath)) ResponseError(c, "could not unescape attr_val: "+err.Error(), fasthttp.StatusBadRequest) return } @@ -271,7 +275,7 @@ func (h *Handler) byAttribute(c *fasthttp.RequestCtx, handler func(context.Conte func (h *Handler) findObjectByAttribute(ctx context.Context, log *zap.Logger, cnrID cid.ID, attrKey, attrVal string) (oid.ID, error) { res, err := h.search(ctx, cnrID, attrKey, attrVal, object.MatchStringEqual) if err != nil { - log.Error(logs.CouldNotSearchForObjects, zap.Error(err)) + log.Error(logs.CouldNotSearchForObjects, zap.Error(err), logs.TagField(logs.TagExternalStorage)) return oid.ID{}, fmt.Errorf("could not search for objects: %w", err) } defer res.Close() @@ -282,13 +286,13 @@ func (h *Handler) findObjectByAttribute(ctx context.Context, log *zap.Logger, cn if n == 0 { switch { case errors.Is(err, io.EOF) && h.needSearchByFileName(attrKey, attrVal): - log.Debug(logs.ObjectNotFoundByFilePathTrySearchByFileName) + log.Debug(logs.ObjectNotFoundByFilePathTrySearchByFileName, logs.TagField(logs.TagExternalStorage)) return h.findObjectByAttribute(ctx, log, cnrID, attrFileName, attrVal) case errors.Is(err, io.EOF): - log.Error(logs.ObjectNotFound, zap.Error(err)) + log.Error(logs.ObjectNotFound, zap.Error(err), logs.TagField(logs.TagExternalStorage)) return oid.ID{}, fmt.Errorf("object not found: %w", err) default: - log.Error(logs.ReadObjectListFailed, zap.Error(err)) + log.Error(logs.ReadObjectListFailed, zap.Error(err), logs.TagField(logs.TagExternalStorage)) return oid.ID{}, fmt.Errorf("read object list failed: %w", err) } } @@ -330,11 +334,16 @@ func (h *Handler) getBucketInfo(ctx context.Context, containerName string, log * cnrID, err := h.resolveContainer(ctx, containerName) if err != nil { + log.Error(logs.CouldNotResolveContainerID, zap.Error(err), zap.String("cnrName", containerName), + logs.TagField(logs.TagDatapath)) return nil, err } bktInfo, err := h.readContainer(ctx, *cnrID) if err != nil { + log.Error(logs.CouldNotGetContainerInfo, zap.Error(err), zap.String("cnrName", containerName), + zap.String("cnrName", cnrID.String()), + logs.TagField(logs.TagExternalStorage)) return nil, err } @@ -342,7 +351,8 @@ func (h *Handler) getBucketInfo(ctx context.Context, containerName string, log * log.Warn(logs.CouldntPutBucketIntoCache, zap.String("bucket name", bktInfo.Name), zap.Stringer("bucket cid", bktInfo.CID), - zap.Error(err)) + zap.Error(err), + logs.TagField(logs.TagDatapath)) } return bktInfo, nil diff --git a/internal/handler/head.go b/internal/handler/head.go index 94f5ccb..0b5adc4 100644 --- a/internal/handler/head.go +++ b/internal/handler/head.go @@ -67,7 +67,8 @@ func (h *Handler) headObject(ctx context.Context, req request, objectAddress oid req.log.Info(logs.CouldntParseCreationDate, zap.String("key", key), zap.String("val", val), - zap.Error(err)) + zap.Error(err), + logs.TagField(logs.TagDatapath)) continue } req.Response.Header.Set(fasthttp.HeaderLastModified, time.Unix(value, 0).UTC().Format(http.TimeFormat)) @@ -131,6 +132,8 @@ func (h *Handler) HeadByAddressOrBucketName(c *fasthttp.RequestCtx) { } checkS3Err := h.tree.CheckSettingsNodeExists(ctx, bktInfo) if checkS3Err != nil && !errors.Is(checkS3Err, layer.ErrNodeNotFound) { + log.Error(logs.FailedToCheckIfSettingsNodeExist, zap.String("cid", bktInfo.CID.String()), + zap.Error(checkS3Err), logs.TagField(logs.TagExternalStorageTree)) logAndSendBucketError(c, log, checkS3Err) return } @@ -144,7 +147,6 @@ func (h *Handler) HeadByAddressOrBucketName(c *fasthttp.RequestCtx) { h.byNativeAddress(ctx, req, bktInfo.CID, objID, h.headObject) } else { logAndSendBucketError(c, log, checkS3Err) - return } } diff --git a/internal/handler/multipart.go b/internal/handler/multipart.go index ebf5edd..5ed2350 100644 --- a/internal/handler/multipart.go +++ b/internal/handler/multipart.go @@ -33,7 +33,7 @@ func fetchMultipartFile(l *zap.Logger, r io.Reader, boundary string) (MultipartF name := part.FormName() if name == "" { - l.Debug(logs.IgnorePartEmptyFormName) + l.Debug(logs.IgnorePartEmptyFormName, logs.TagField(logs.TagDatapath)) continue } @@ -41,9 +41,9 @@ func fetchMultipartFile(l *zap.Logger, r io.Reader, boundary string) (MultipartF // ignore multipart/form-data values if filename == "" { - l.Debug(logs.IgnorePartEmptyFilename, zap.String("form", name)) + l.Debug(logs.IgnorePartEmptyFilename, zap.String("form", name), logs.TagField(logs.TagDatapath)) if err = part.Close(); err != nil { - l.Warn(logs.FailedToCloseReader, zap.Error(err)) + l.Warn(logs.FailedToCloseReader, zap.Error(err), logs.TagField(logs.TagDatapath)) } continue } diff --git a/internal/handler/multipart_test.go b/internal/handler/multipart_test.go index 2c50a87..431d0d6 100644 --- a/internal/handler/multipart_test.go +++ b/internal/handler/multipart_test.go @@ -112,7 +112,7 @@ func fetchMultipartFileDefault(l *zap.Logger, r io.Reader, boundary string) (Mul name := part.FormName() if name == "" { - l.Debug(logs.IgnorePartEmptyFormName) + l.Debug(logs.IgnorePartEmptyFormName, logs.TagField(logs.TagDatapath)) continue } @@ -120,8 +120,7 @@ func fetchMultipartFileDefault(l *zap.Logger, r io.Reader, boundary string) (Mul // ignore multipart/form-data values if filename == "" { - l.Debug(logs.IgnorePartEmptyFilename, zap.String("form", name)) - + l.Debug(logs.IgnorePartEmptyFilename, zap.String("form", name), logs.TagField(logs.TagDatapath)) continue } diff --git a/internal/handler/reader.go b/internal/handler/reader.go index cbd8294..e8ac098 100644 --- a/internal/handler/reader.go +++ b/internal/handler/reader.go @@ -110,7 +110,8 @@ func (h *Handler) receiveFile(ctx context.Context, req request, objAddress oid.A if err = req.setTimestamp(val); err != nil { req.log.Error(logs.CouldntParseCreationDate, zap.String("val", val), - zap.Error(err)) + zap.Error(err), + logs.TagField(logs.TagDatapath)) } case object.AttributeContentType: contentType = val @@ -144,7 +145,7 @@ func (h *Handler) receiveFile(ctx context.Context, req request, objAddress oid.A return payload, nil }, filename) if err != nil && err != io.EOF { - req.log.Error(logs.CouldNotDetectContentTypeFromPayload, zap.Error(err)) + req.log.Error(logs.CouldNotDetectContentTypeFromPayload, zap.Error(err), logs.TagField(logs.TagDatapath)) ResponseError(req.RequestCtx, "could not detect Content-Type from payload: "+err.Error(), fasthttp.StatusBadRequest) return } diff --git a/internal/handler/upload.go b/internal/handler/upload.go index d1953c9..272e911 100644 --- a/internal/handler/upload.go +++ b/internal/handler/upload.go @@ -68,14 +68,14 @@ func (h *Handler) Upload(c *fasthttp.RequestCtx) { boundary := string(c.Request.Header.MultipartFormBoundary()) if file, err = fetchMultipartFile(log, bodyStream, boundary); err != nil { - log.Error(logs.CouldNotReceiveMultipartForm, zap.Error(err)) + log.Error(logs.CouldNotReceiveMultipartForm, zap.Error(err), logs.TagField(logs.TagDatapath)) ResponseError(c, "could not receive multipart/form: "+err.Error(), fasthttp.StatusBadRequest) return } filtered, err := filterHeaders(log, &c.Request.Header) if err != nil { - log.Error(logs.FailedToFilterHeaders, zap.Error(err)) + log.Error(logs.FailedToFilterHeaders, zap.Error(err), logs.TagField(logs.TagDatapath)) ResponseError(c, err.Error(), fasthttp.StatusBadRequest) return } @@ -106,7 +106,7 @@ func (h *Handler) uploadSingleObject(req request, bkt *data.BucketInfo, file Mul attributes, err := h.extractAttributes(c, log, filtered) if err != nil { - log.Error(logs.FailedToGetAttributes, zap.Error(err)) + log.Error(logs.FailedToGetAttributes, zap.Error(err), logs.TagField(logs.TagDatapath)) ResponseError(c, "could not extract attributes: "+err.Error(), fasthttp.StatusBadRequest) return } @@ -119,13 +119,14 @@ func (h *Handler) uploadSingleObject(req request, bkt *data.BucketInfo, file Mul log.Debug(logs.ObjectUploaded, zap.String("oid", idObj.EncodeToString()), zap.String("FileName", file.FileName()), + logs.TagField(logs.TagExternalStorage), ) addr := newAddress(bkt.CID, idObj) c.Response.Header.SetContentType(jsonHeader) // Try to return the response, otherwise, if something went wrong, throw an error. if err = newPutResponse(addr).encode(c); err != nil { - log.Error(logs.CouldNotEncodeResponse, zap.Error(err)) + log.Error(logs.CouldNotEncodeResponse, zap.Error(err), logs.TagField(logs.TagDatapath)) ResponseError(c, "could not encode response", fasthttp.StatusBadRequest) return } @@ -162,13 +163,14 @@ func (h *Handler) extractAttributes(c *fasthttp.RequestCtx, log *zap.Logger, fil now := time.Now() if rawHeader := c.Request.Header.Peek(fasthttp.HeaderDate); rawHeader != nil { if parsed, err := time.Parse(http.TimeFormat, string(rawHeader)); err != nil { - log.Warn(logs.CouldNotParseClientTime, zap.String("Date header", string(rawHeader)), zap.Error(err)) + log.Warn(logs.CouldNotParseClientTime, zap.String("Date header", string(rawHeader)), zap.Error(err), + logs.TagField(logs.TagDatapath)) } else { now = parsed } } if err := utils.PrepareExpirationHeader(c, h.frostfs, filtered, now); err != nil { - log.Error(logs.CouldNotPrepareExpirationHeader, zap.Error(err)) + log.Error(logs.CouldNotPrepareExpirationHeader, zap.Error(err), logs.TagField(logs.TagDatapath)) return nil, err } attributes := make([]object.Attribute, 0, len(filtered)) @@ -205,7 +207,7 @@ func (h *Handler) explodeArchive(req request, bkt *data.BucketInfo, file io.Read commonAttributes, err := h.extractAttributes(c, log, filtered) if err != nil { - log.Error(logs.FailedToGetAttributes, zap.Error(err)) + log.Error(logs.FailedToGetAttributes, zap.Error(err), logs.TagField(logs.TagDatapath)) ResponseError(c, "could not extract attributes: "+err.Error(), fasthttp.StatusBadRequest) return } @@ -213,16 +215,16 @@ func (h *Handler) explodeArchive(req request, bkt *data.BucketInfo, file io.Read reader := file if bytes.EqualFold(c.Request.Header.Peek(fasthttp.HeaderContentEncoding), []byte("gzip")) { - log.Debug(logs.GzipReaderSelected) + log.Debug(logs.GzipReaderSelected, logs.TagField(logs.TagDatapath)) gzipReader, err := gzip.NewReader(file) if err != nil { - log.Error(logs.FailedToCreateGzipReader, zap.Error(err)) + log.Error(logs.FailedToCreateGzipReader, zap.Error(err), logs.TagField(logs.TagDatapath)) ResponseError(c, "could read gzip file: "+err.Error(), fasthttp.StatusBadRequest) return } defer func() { if err := gzipReader.Close(); err != nil { - log.Warn(logs.FailedToCloseReader, zap.Error(err)) + log.Warn(logs.FailedToCloseReader, zap.Error(err), logs.TagField(logs.TagDatapath)) } }() reader = gzipReader @@ -234,7 +236,7 @@ func (h *Handler) explodeArchive(req request, bkt *data.BucketInfo, file io.Read if errors.Is(err, io.EOF) { break } else if err != nil { - log.Error(logs.FailedToReadFileFromTar, zap.Error(err)) + log.Error(logs.FailedToReadFileFromTar, zap.Error(err), logs.TagField(logs.TagDatapath)) ResponseError(c, "could not get next entry: "+err.Error(), fasthttp.StatusBadRequest) return } @@ -258,6 +260,7 @@ func (h *Handler) explodeArchive(req request, bkt *data.BucketInfo, file io.Read log.Debug(logs.ObjectUploaded, zap.String("oid", idObj.EncodeToString()), zap.String("FileName", fileName), + logs.TagField(logs.TagExternalStorage), ) } } @@ -266,7 +269,7 @@ func (h *Handler) handlePutFrostFSErr(r *fasthttp.RequestCtx, err error, log *za statusCode, msg, additionalFields := formErrorResponse("could not store file in frostfs", err) logFields := append([]zap.Field{zap.Error(err)}, additionalFields...) - log.Error(logs.CouldNotStoreFileInFrostfs, logFields...) + log.Error(logs.CouldNotStoreFileInFrostfs, append(logFields, logs.TagField(logs.TagExternalStorage))...) ResponseError(r, msg, statusCode) } diff --git a/internal/handler/utils.go b/internal/handler/utils.go index 74932f3..0a1dc62 100644 --- a/internal/handler/utils.go +++ b/internal/handler/utils.go @@ -39,7 +39,7 @@ func (r *request) handleFrostFSErr(err error, start time.Time) { statusCode, msg, additionalFields := formErrorResponse("could not receive object", err) logFields = append(logFields, additionalFields...) - r.log.Error(logs.CouldNotReceiveObject, logFields...) + r.log.Error(logs.CouldNotReceiveObject, append(logFields, logs.TagField(logs.TagExternalStorage))...) ResponseError(r.RequestCtx, msg, statusCode) } @@ -85,7 +85,7 @@ func isValidValue(s string) bool { } func logAndSendBucketError(c *fasthttp.RequestCtx, log *zap.Logger, err error) { - log.Error(logs.CouldntGetBucket, zap.Error(err)) + log.Error(logs.CouldNotGetBucket, zap.Error(err), logs.TagField(logs.TagDatapath)) if client.IsErrContainerNotFound(err) { ResponseError(c, "Not Found", fasthttp.StatusNotFound) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 5f60b9b..072e9c9 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -1,63 +1,43 @@ package logs +import "go.uber.org/zap" + +const ( + TagFieldName = "tag" + + TagApp = "app" + TagDatapath = "datapath" + TagExternalStorage = "external_storage" + TagExternalStorageTree = "external_storage_tree" +) + +func TagField(tag string) zap.Field { + return zap.String(TagFieldName, tag) +} + +// Log messages with the "app" tag. const ( - CouldntParseCreationDate = "couldn't parse creation date" - CouldNotDetectContentTypeFromPayload = "could not detect Content-Type from payload" - CouldNotReceiveObject = "could not receive object" - ObjectWasDeleted = "object was deleted" - CouldNotSearchForObjects = "could not search for objects" - ObjectNotFound = "object not found" - ReadObjectListFailed = "read object list failed" - FailedToAddObjectToArchive = "failed to add object to archive" - FailedToGetObject = "failed to get object" - IteratingOverSelectedObjectsFailed = "iterating over selected objects failed" - ObjectsNotFound = "objects not found" - CloseZipWriter = "close zip writer" ServiceIsRunning = "service is running" ServiceCouldntStartOnConfiguredPort = "service couldn't start on configured port" ServiceHasntStartedSinceItsDisabled = "service hasn't started since it's disabled" ShuttingDownService = "shutting down service" CantShutDownService = "can't shut down service" CantGracefullyShutDownService = "can't gracefully shut down service, force stop" - IgnorePartEmptyFormName = "ignore part, empty form name" - IgnorePartEmptyFilename = "ignore part, empty filename" - CouldNotReceiveMultipartForm = "could not receive multipart/form" - CouldNotParseClientTime = "could not parse client time" - CouldNotPrepareExpirationHeader = "could not prepare expiration header" - CouldNotEncodeResponse = "could not encode response" - CouldNotStoreFileInFrostfs = "could not store file in frostfs" - AddAttributeToResultObject = "add attribute to result object" FailedToCreateResolver = "failed to create resolver" FailedToCreateWorkerPool = "failed to create worker pool" - FailedToReadIndexPageTemplate = "failed to read index page template" - SetCustomIndexPageTemplate = "set custom index page template" - ContainerResolverWillBeDisabledBecauseOfResolversResolverOrderIsEmpty = "container resolver will be disabled because of resolvers 'resolver_order' is empty" - MetricsAreDisabled = "metrics are disabled" - NoWalletPathSpecifiedCreatingEphemeralKeyAutomaticallyForThisRun = "no wallet path specified, creating ephemeral key automatically for this run" StartingApplication = "starting application" StartingServer = "starting server" ListenAndServe = "listen and serve" ShuttingDownWebServer = "shutting down web server" FailedToShutdownTracing = "failed to shutdown tracing" - SIGHUPConfigReloadStarted = "SIGHUP config reload started" - FailedToReloadConfigBecauseItsMissed = "failed to reload config because it's missed" - FailedToReloadConfig = "failed to reload config" - LogLevelWontBeUpdated = "log level won't be updated" - FailedToUpdateResolvers = "failed to update resolvers" - FailedToReloadServerParameters = "failed to reload server parameters" - SIGHUPConfigReloadCompleted = "SIGHUP config reload completed" AddedPathUploadCid = "added path /upload/{cid}" AddedPathGetCidOid = "added path /get/{cid}/{oid}" AddedPathGetByAttributeCidAttrKeyAttrVal = "added path /get_by_attribute/{cid}/{attr_key}/{attr_val:*}" AddedPathZipCidPrefix = "added path /zip/{cid}/{prefix}" - Request = "request" - CouldNotFetchAndStoreBearerToken = "could not fetch and store bearer token" FailedToAddServer = "failed to add server" AddServer = "add server" NoHealthyServers = "no healthy servers" FailedToInitializeTracing = "failed to initialize tracing" - TracingConfigUpdated = "tracing config updated" - ResolverNNSWontBeUsedSinceRPCEndpointIsntProvided = "resolver nns won't be used since rpc_endpoint isn't provided" RuntimeSoftMemoryDefinedWithGOMEMLIMIT = "soft runtime memory defined with GOMEMLIMIT environment variable, config value skipped" RuntimeSoftMemoryLimitUpdated = "soft runtime memory limit value updated" CouldNotLoadFrostFSPrivateKey = "could not load FrostFS private key" @@ -66,33 +46,86 @@ const ( FailedToDialConnectionPool = "failed to dial connection pool" FailedToCreateTreePool = "failed to create tree pool" FailedToDialTreePool = "failed to dial tree pool" - AddedStoragePeer = "added storage peer" - CouldntGetBucket = "could not get bucket" - CouldntPutBucketIntoCache = "couldn't put bucket info into cache" - FailedToSumbitTaskToPool = "failed to submit task to pool" - FailedToHeadObject = "failed to head object" - FailedToIterateOverResponse = "failed to iterate over search response" - InvalidCacheEntryType = "invalid cache entry type" - InvalidLifetimeUsingDefaultValue = "invalid lifetime, using default value (in seconds)" - InvalidCacheSizeUsingDefaultValue = "invalid cache size, using default value" - FailedToUnescapeQuery = "failed to unescape query" ServerReconnecting = "reconnecting server..." ServerReconnectedSuccessfully = "server reconnected successfully" ServerReconnectFailed = "failed to reconnect server" - WarnDuplicateAddress = "duplicate address" + FailedToSumbitTaskToPool = "failed to submit task to pool" MultinetDialSuccess = "multinet dial successful" MultinetDialFail = "multinet dial failed" + ContainerResolverWillBeDisabledBecauseOfResolversResolverOrderIsEmpty = "container resolver will be disabled because of resolvers 'resolver_order' is empty" + MetricsAreDisabled = "metrics are disabled" + NoWalletPathSpecifiedCreatingEphemeralKeyAutomaticallyForThisRun = "no wallet path specified, creating ephemeral key automatically for this run" + SIGHUPConfigReloadStarted = "SIGHUP config reload started" + FailedToReloadConfigBecauseItsMissed = "failed to reload config because it's missed" + FailedToReloadConfig = "failed to reload config" + FailedToUpdateResolvers = "failed to update resolvers" + FailedToReloadServerParameters = "failed to reload server parameters" + SIGHUPConfigReloadCompleted = "SIGHUP config reload completed" + TracingConfigUpdated = "tracing config updated" + ResolverNNSWontBeUsedSinceRPCEndpointIsntProvided = "resolver nns won't be used since rpc_endpoint isn't provided" + AddedStoragePeer = "added storage peer" + InvalidLifetimeUsingDefaultValue = "invalid lifetime, using default value (in seconds)" + InvalidCacheSizeUsingDefaultValue = "invalid cache size, using default value" + WarnDuplicateAddress = "duplicate address" FailedToLoadMultinetConfig = "failed to load multinet config" MultinetConfigWontBeUpdated = "multinet config won't be updated" - ObjectNotFoundByFilePathTrySearchByFileName = "object not found by filePath attribute, try search by fileName" - CouldntCacheNetmap = "couldn't cache netmap" - FailedToFilterHeaders = "failed to filter headers" - FailedToReadFileFromTar = "failed to read file from tar" - FailedToGetAttributes = "failed to get attributes" - ObjectUploaded = "object uploaded" - CloseGzipWriter = "close gzip writer" - CloseTarWriter = "close tar writer" - FailedToCloseReader = "failed to close reader" - FailedToCreateGzipReader = "failed to create gzip reader" - GzipReaderSelected = "gzip reader selected" + LogLevelWontBeUpdated = "log level won't be updated" + TagsLogConfigWontBeUpdated = "tags log config won't be updated" + FailedToReadIndexPageTemplate = "failed to read index page template" + SetCustomIndexPageTemplate = "set custom index page template" +) + +// Log messages with the "datapath" tag. +const ( + CouldntParseCreationDate = "couldn't parse creation date" + CouldNotDetectContentTypeFromPayload = "could not detect Content-Type from payload" + FailedToAddObjectToArchive = "failed to add object to archive" + CloseZipWriter = "close zip writer" + IgnorePartEmptyFormName = "ignore part, empty form name" + IgnorePartEmptyFilename = "ignore part, empty filename" + CouldNotParseClientTime = "could not parse client time" + CouldNotPrepareExpirationHeader = "could not prepare expiration header" + CouldNotEncodeResponse = "could not encode response" + AddAttributeToResultObject = "add attribute to result object" + Request = "request" + CouldNotFetchAndStoreBearerToken = "could not fetch and store bearer token" + CouldntPutBucketIntoCache = "couldn't put bucket info into cache" + FailedToIterateOverResponse = "failed to iterate over search response" + InvalidCacheEntryType = "invalid cache entry type" + FailedToUnescapeQuery = "failed to unescape query" + CouldntCacheNetmap = "couldn't cache netmap" + FailedToCloseReader = "failed to close reader" + FailedToFilterHeaders = "failed to filter headers" + FailedToReadFileFromTar = "failed to read file from tar" + FailedToGetAttributes = "failed to get attributes" + CloseGzipWriter = "close gzip writer" + CloseTarWriter = "close tar writer" + FailedToCreateGzipReader = "failed to create gzip reader" + GzipReaderSelected = "gzip reader selected" + CouldNotReceiveMultipartForm = "could not receive multipart/form" + ObjectsNotFound = "objects not found" + IteratingOverSelectedObjectsFailed = "iterating over selected objects failed" + CouldNotGetBucket = "could not get bucket" + CouldNotResolveContainerID = "could not resolve container id" +) + +// Log messages with the "external_storage" tag. +const ( + CouldNotReceiveObject = "could not receive object" + CouldNotSearchForObjects = "could not search for objects" + ObjectNotFound = "object not found" + ReadObjectListFailed = "read object list failed" + CouldNotStoreFileInFrostfs = "could not store file in frostfs" + FailedToHeadObject = "failed to head object" + ObjectNotFoundByFilePathTrySearchByFileName = "object not found by filePath attribute, try search by fileName" + FailedToGetObject = "failed to get object" + ObjectUploaded = "object uploaded" + CouldNotGetContainerInfo = "could not get container info" +) + +// Log messages with the "external_storage_tree" tag. +const ( + ObjectWasDeleted = "object was deleted" + FailedToGetLatestVersionOfObject = "failed to get latest version of object" + FailedToCheckIfSettingsNodeExist = "Failed to check if settings node exists" ) diff --git a/internal/net/event_handler.go b/internal/net/event_handler.go index 9520c01..2826d35 100644 --- a/internal/net/event_handler.go +++ b/internal/net/event_handler.go @@ -17,9 +17,11 @@ func (l LogEventHandler) DialPerformed(sourceIP net.Addr, _, address string, err sourceIPString = sourceIP.Network() + "://" + sourceIP.String() } if err == nil { - l.logger.Debug(logs.MultinetDialSuccess, zap.String("source", sourceIPString), zap.String("destination", address)) + l.logger.Debug(logs.MultinetDialSuccess, zap.String("source", sourceIPString), + zap.String("destination", address), logs.TagField(logs.TagApp)) } else { - l.logger.Debug(logs.MultinetDialFail, zap.String("source", sourceIPString), zap.String("destination", address), zap.Error(err)) + l.logger.Debug(logs.MultinetDialFail, zap.String("source", sourceIPString), + zap.String("destination", address), logs.TagField(logs.TagApp)) } } diff --git a/internal/service/frostfs/source.go b/internal/service/frostfs/source.go index de6c681..84f7b74 100644 --- a/internal/service/frostfs/source.go +++ b/internal/service/frostfs/source.go @@ -40,7 +40,7 @@ func (s *Source) NetMapSnapshot(ctx context.Context) (netmap.NetMap, error) { } if err = s.netmapCache.Put(netmapSnapshot); err != nil { - s.log.Warn(logs.CouldntCacheNetmap, zap.Error(err)) + s.log.Warn(logs.CouldntCacheNetmap, zap.Error(err), logs.TagField(logs.TagDatapath)) } return netmapSnapshot, nil diff --git a/metrics/service.go b/metrics/service.go index dea5ac0..e6b803b 100644 --- a/metrics/service.go +++ b/metrics/service.go @@ -25,24 +25,24 @@ type Config struct { // Start runs http service with the exposed endpoint on the configured port. func (ms *Service) Start() { if ms.enabled { - ms.log.Info(logs.ServiceIsRunning, zap.String("endpoint", ms.Addr)) + ms.log.Info(logs.ServiceIsRunning, zap.String("endpoint", ms.Addr), logs.TagField(logs.TagApp)) err := ms.ListenAndServe() if err != nil && err != http.ErrServerClosed { - ms.log.Warn(logs.ServiceCouldntStartOnConfiguredPort) + ms.log.Warn(logs.ServiceCouldntStartOnConfiguredPort, logs.TagField(logs.TagApp)) } } else { - ms.log.Info(logs.ServiceHasntStartedSinceItsDisabled) + ms.log.Info(logs.ServiceHasntStartedSinceItsDisabled, logs.TagField(logs.TagApp)) } } // ShutDown stops the service. func (ms *Service) ShutDown(ctx context.Context) { - ms.log.Info(logs.ShuttingDownService, zap.String("endpoint", ms.Addr)) + ms.log.Info(logs.ShuttingDownService, zap.String("endpoint", ms.Addr), logs.TagField(logs.TagApp)) err := ms.Shutdown(ctx) if err != nil { - ms.log.Error(logs.CantGracefullyShutDownService, zap.Error(err)) + ms.log.Error(logs.CantGracefullyShutDownService, zap.Error(err), logs.TagField(logs.TagApp)) if err = ms.Close(); err != nil { - ms.log.Panic(logs.CantShutDownService, zap.Error(err)) + ms.log.Panic(logs.CantShutDownService, zap.Error(err), logs.TagField(logs.TagApp)) } } }