[#559] Remove multipart objects using tombstones

Signed-off-by: Marina Biryukova <m.biryukova@yadro.com>

CHANGELOG.md

@@ -4,12 +4,6 @@ This document outlines major changes between releases.
 
 ## [Unreleased]
 
-## [0.31.1] - 2024-11-28
-
-### Fixed
-- Ignore precondition headers with invalid date format (#563)
-- MD5 calculation of object-part with SSE-C (#543)
-
 ## [0.31.0] - Rongbuk - 2024-11-20
 
 ### Fixed
@@ -348,5 +342,4 @@ To see CHANGELOG for older versions, refer to https://github.com/nspcc-dev/neofs
 [0.30.7]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.30.6...v0.30.7
 [0.30.8]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.30.7...v0.30.8
 [0.31.0]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.30.8...v0.31.0
-[0.31.1]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.31.0...v0.31.1
-[Unreleased]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.31.1...master
+[Unreleased]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.31.0...master

VERSION

@@ -1 +1 @@
-v0.31.1
+v0.31.0

cmd/s3-gw/app.go

@@ -84,17 +84,11 @@ type (
 		bucketResolver *resolver.BucketResolver
 		services       []*Service
 		settings       *appSettings
-		loggerSettings *loggerSettings
 
 		webDone chan struct{}
 		wrkDone chan struct{}
 	}
 
-	loggerSettings struct {
-		mu         sync.RWMutex
-		appMetrics *metrics.AppMetrics
-	}
-
 	appSettings struct {
 		logLevel            zap.AtomicLevel
 		httpLogging         s3middleware.LogHTTPConfig
@@ -142,25 +136,7 @@ type (
 	}
 )
 
-func (s *loggerSettings) DroppedLogsInc() {
-	s.mu.RLock()
-	defer s.mu.RUnlock()
-
-	if s.appMetrics != nil {
-		s.appMetrics.Statistic().DroppedLogsInc()
-	}
-}
-
-func (s *loggerSettings) setMetrics(appMetrics *metrics.AppMetrics) {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-
-	s.appMetrics = appMetrics
-}
-
-func newApp(ctx context.Context, v *viper.Viper) *App {
-	logSettings := &loggerSettings{}
-	log := pickLogger(v, logSettings)
+func newApp(ctx context.Context, log *Logger, v *viper.Viper) *App {
 	settings := newAppSettings(log, v)
 
 	objPool, treePool, key := getPools(ctx, log.logger, v, settings.dialerSource)
@@ -175,8 +151,7 @@ func newApp(ctx context.Context, v *viper.Viper) *App {
 		webDone: make(chan struct{}, 1),
 		wrkDone: make(chan struct{}, 1),
 
-		settings:       settings,
-		loggerSettings: logSettings,
+		settings: settings,
 	}
 
 	app.init(ctx)
@@ -576,7 +551,6 @@ func (a *App) initMetrics() {
 
 	a.metrics = metrics.NewAppMetrics(cfg)
 	a.metrics.State().SetHealth(metrics.HealthStatusStarting)
-	a.loggerSettings.setMetrics(a.metrics)
 }
 
 func (a *App) initFrostfsID(ctx context.Context) {

cmd/s3-gw/app_settings.go

@@ -1131,11 +1131,7 @@ func mergeConfig(v *viper.Viper, fileName string) error {
 	return v.MergeConfig(cfgFile)
 }
 
-type LoggerAppSettings interface {
-	DroppedLogsInc()
-}
-
-func pickLogger(v *viper.Viper, settings LoggerAppSettings) *Logger {
+func pickLogger(v *viper.Viper) *Logger {
 	lvl, err := getLogLevel(v)
 	if err != nil {
 		panic(err)
@@ -1145,9 +1141,9 @@ func pickLogger(v *viper.Viper, settings LoggerAppSettings) *Logger {
 
 	switch dest {
 	case destinationStdout:
-		return newStdoutLogger(v, lvl, settings)
+		return newStdoutLogger(v, lvl)
 	case destinationJournald:
-		return newJournaldLogger(v, lvl, settings)
+		return newJournaldLogger(v, lvl)
 	default:
 		panic(fmt.Sprintf("wrong destination for logger: %s", dest))
 	}
@@ -1167,13 +1163,13 @@ func pickLogger(v *viper.Viper, settings LoggerAppSettings) *Logger {
 // Logger records a stack trace for all messages at or above fatal level.
 //
 // See also zapcore.Level, zap.NewProductionConfig, zap.AddStacktrace.
-func newStdoutLogger(v *viper.Viper, lvl zapcore.Level, settings LoggerAppSettings) *Logger {
+func newStdoutLogger(v *viper.Viper, lvl zapcore.Level) *Logger {
 	stdout := zapcore.AddSync(os.Stderr)
 	level := zap.NewAtomicLevelAt(lvl)
 
 	consoleOutCore := zapcore.NewCore(newLogEncoder(), stdout, level)
 
-	consoleOutCore = applyZapCoreMiddlewares(consoleOutCore, v, settings)
+	consoleOutCore = samplingEnabling(v, consoleOutCore)
 
 	return &Logger{
 		logger: zap.New(consoleOutCore, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel))),
@@ -1181,7 +1177,7 @@ func newStdoutLogger(v *viper.Viper, lvl zapcore.Level, settings LoggerAppSettin
 	}
 }
 
-func newJournaldLogger(v *viper.Viper, lvl zapcore.Level, settings LoggerAppSettings) *Logger {
+func newJournaldLogger(v *viper.Viper, lvl zapcore.Level) *Logger {
 	level := zap.NewAtomicLevelAt(lvl)
 
 	encoder := zapjournald.NewPartialEncoder(newLogEncoder(), zapjournald.SyslogFields)
@@ -1193,7 +1189,7 @@ func newJournaldLogger(v *viper.Viper, lvl zapcore.Level, settings LoggerAppSett
 		zapjournald.SyslogPid(),
 	})
 
-	coreWithContext = applyZapCoreMiddlewares(coreWithContext, v, settings)
+	coreWithContext = samplingEnabling(v, coreWithContext)
 
 	l := zap.New(coreWithContext, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel)))
 
@@ -1210,17 +1206,19 @@ func newLogEncoder() zapcore.Encoder {
 	return zapcore.NewConsoleEncoder(c)
 }
 
-func applyZapCoreMiddlewares(core zapcore.Core, v *viper.Viper, settings LoggerAppSettings) zapcore.Core {
+func samplingEnabling(v *viper.Viper, core zapcore.Core) zapcore.Core {
+	// Zap samples by logging the first cgfLoggerSamplingInitial entries with a given level
+	// and message within the specified time interval.
+	// In the above config, only the first cgfLoggerSamplingInitial log entries with the same level and message
+	// are recorded in cfgLoggerSamplingInterval interval. Every other log entry will be dropped within the interval since
+	// cfgLoggerSamplingThereafter is specified here.
 	if v.GetBool(cfgLoggerSamplingEnabled) {
-		core = zapcore.NewSamplerWithOptions(core,
+		core = zapcore.NewSamplerWithOptions(
+			core,
 			v.GetDuration(cfgLoggerSamplingInterval),
 			v.GetInt(cfgLoggerSamplingInitial),
 			v.GetInt(cfgLoggerSamplingThereafter),
-			zapcore.SamplerHook(func(_ zapcore.Entry, dec zapcore.SamplingDecision) {
-				if dec&zapcore.LogDropped > 0 {
-					settings.DroppedLogsInc()
-				}
-			}))
+		)
 	}
 
 	return core

cmd/s3-gw/main.go

@@ -8,9 +8,11 @@ import (
 
 func main() {
 	g, _ := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
-	v := newSettings()
 
-	a := newApp(g, v)
+	v := newSettings()
+	l := pickLogger(v)
+
+	a := newApp(g, l, v)
 
 	go a.Serve(g)
 

metrics/desc.go

@@ -93,13 +93,6 @@ var appMetricsDesc = map[string]map[string]Description{
 		},
 	},
 	statisticSubsystem: {
-		droppedLogs: Description{
-			Type:      dto.MetricType_COUNTER,
-			Namespace: namespace,
-			Subsystem: statisticSubsystem,
-			Name:      droppedLogs,
-			Help:      "Dropped logs (by sampling) count",
-		},
 		requestsSecondsMetric: Description{
 			Type:           dto.MetricType_HISTOGRAM,
 			Namespace:      namespace,
@@ -259,12 +252,3 @@ func mustNewHistogramVec(description Description, buckets []float64) *prometheus
 		description.VariableLabels,
 	)
 }
-
-func mustNewCounter(description Description) prometheus.Counter {
-	if description.Type != dto.MetricType_COUNTER {
-		panic("invalid metric type")
-	}
-	return prometheus.NewCounter(
-		prometheus.CounterOpts(newOpts(description)),
-	)
-}

metrics/stats.go

@@ -34,7 +34,6 @@ type (
 	APIStatMetrics struct {
 		stats                *httpStats
 		httpRequestsDuration *prometheus.HistogramVec
-		droppedLogs          prometheus.Counter
 	}
 )
 
@@ -48,7 +47,6 @@ const (
 	requestsTotalMetric   = "requests_total"
 	errorsTotalMetric     = "errors_total"
 	bytesTotalMetric      = "bytes_total"
-	droppedLogs           = "dropped_logs"
 )
 
 const (
@@ -63,7 +61,6 @@ func newAPIStatMetrics() *APIStatMetrics {
 		stats: newHTTPStats(),
 		httpRequestsDuration: mustNewHistogramVec(histogramDesc,
 			[]float64{.05, .1, .25, .5, 1, 2.5, 5, 10}),
-		droppedLogs: mustNewCounter(appMetricsDesc[statisticSubsystem][droppedLogs]),
 	}
 }
 
@@ -122,7 +119,6 @@ func (a *APIStatMetrics) Describe(ch chan<- *prometheus.Desc) {
 		return
 	}
 	a.stats.Describe(ch)
-	a.droppedLogs.Describe(ch)
 	a.httpRequestsDuration.Describe(ch)
 }
 
@@ -131,17 +127,9 @@ func (a *APIStatMetrics) Collect(ch chan<- prometheus.Metric) {
 		return
 	}
 	a.stats.Collect(ch)
-	a.droppedLogs.Collect(ch)
 	a.httpRequestsDuration.Collect(ch)
 }
 
-func (a *APIStatMetrics) DroppedLogsInc() {
-	if a == nil {
-		return
-	}
-	a.droppedLogs.Inc()
-}
-
 func newHTTPStats() *httpStats {
 	return &httpStats{
 		currentS3RequestsDesc: newDesc(appMetricsDesc[statisticSubsystem][requestsCurrentMetric]),