package main import ( "context" "encoding/hex" "fmt" "net/http" "os" "os/signal" "runtime/debug" "sync" "sync/atomic" "syscall" "time" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" grpctracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/cache" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/handler" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/notifications" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/resolver" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/services" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/version" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/wallet" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/xml" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/metrics" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/pkg/service/tree" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" treepool "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/spf13/viper" "go.uber.org/zap" "google.golang.org/grpc" ) type ( // App is the main application structure. App struct { ctr auth.Center log *zap.Logger cfg *viper.Viper pool *pool.Pool treePool *treepool.Pool key *keys.PrivateKey nc *notifications.Controller obj layer.Client api api.Handler servers []Server metrics *metrics.AppMetrics bucketResolver *resolver.BucketResolver services []*Service settings *appSettings webDone chan struct{} wrkDone chan struct{} } appSettings struct { logLevel zap.AtomicLevel policies *placementPolicy xmlDecoder *xml.DecoderProvider maxClient maxClientsConfig bypassContentEncodingInChunks atomic.Bool clientCut atomic.Bool md5Enabled atomic.Bool } maxClientsConfig struct { deadline time.Duration count int } Logger struct { logger *zap.Logger lvl zap.AtomicLevel } placementPolicy struct { mu sync.RWMutex defaultPolicy netmap.PlacementPolicy regionMap map[string]netmap.PlacementPolicy copiesNumbers map[string][]uint32 defaultCopiesNumbers []uint32 } ) func newApp(ctx context.Context, log *Logger, v *viper.Viper) *App { objPool, treePool, key := getPools(ctx, log.logger, v) // prepare auth center ctr := auth.New(frostfs.NewAuthmateFrostFS(objPool, key), key, v.GetStringSlice(cfgAllowedAccessKeyIDPrefixes), getAccessBoxCacheConfig(v, log.logger)) app := &App{ ctr: ctr, log: log.logger, cfg: v, pool: objPool, treePool: treePool, key: key, webDone: make(chan struct{}, 1), wrkDone: make(chan struct{}, 1), settings: newAppSettings(log, v), } app.init(ctx) return app } func (a *App) init(ctx context.Context) { a.setRuntimeParameters() a.initAPI(ctx) a.initMetrics() a.initServers(ctx) a.initTracing(ctx) } func (a *App) initLayer(ctx context.Context) { a.initResolver() // prepare random key for anonymous requests randomKey, err := keys.NewPrivateKey() if err != nil { a.log.Fatal(logs.CouldntGenerateRandomKey, zap.Error(err)) } var gateOwner user.ID user.IDFromKey(&gateOwner, a.key.PrivateKey.PublicKey) layerCfg := &layer.Config{ Caches: getCacheOptions(a.cfg, a.log), AnonKey: layer.AnonymousKey{ Key: randomKey, }, GateOwner: gateOwner, Resolver: a.bucketResolver, TreeService: tree.NewTree(services.NewPoolWrapper(a.treePool), a.log), Features: a.settings, } // prepare object layer a.obj = layer.NewLayer(a.log, frostfs.NewFrostFS(a.pool, a.key), layerCfg) if a.cfg.GetBool(cfgEnableNATS) { nopts := getNotificationsOptions(a.cfg, a.log) a.nc, err = notifications.NewController(nopts, a.log) if err != nil { a.log.Fatal(logs.FailedToEnableNotifications, zap.Error(err)) } if err = a.obj.Initialize(ctx, a.nc); err != nil { a.log.Fatal(logs.CouldntInitializeLayer, zap.Error(err)) } } } func newAppSettings(log *Logger, v *viper.Viper) *appSettings { settings := &appSettings{ logLevel: log.lvl, policies: newPlacementPolicy(log.logger, v), xmlDecoder: xml.NewDecoderProvider(v.GetBool(cfgKludgeUseDefaultXMLNSForCompleteMultipartUpload)), maxClient: newMaxClients(v), } settings.setBypassContentEncodingInChunks(v.GetBool(cfgKludgeBypassContentEncodingCheckInChunks)) settings.setClientCut(v.GetBool(cfgClientCut)) settings.setMD5Enabled(v.GetBool(cfgMD5Enabled)) return settings } func (s *appSettings) BypassContentEncodingInChunks() bool { return s.bypassContentEncodingInChunks.Load() } func (s *appSettings) setBypassContentEncodingInChunks(bypass bool) { s.bypassContentEncodingInChunks.Store(bypass) } func (s *appSettings) ClientCut() bool { return s.clientCut.Load() } func (s *appSettings) setClientCut(clientCut bool) { s.clientCut.Store(clientCut) } func (s *appSettings) MD5Enabled() bool { return s.md5Enabled.Load() } func (s *appSettings) setMD5Enabled(md5Enabled bool) { s.md5Enabled.Store(md5Enabled) } func (a *App) initAPI(ctx context.Context) { a.initLayer(ctx) a.initHandler() } func (a *App) initMetrics() { a.metrics = metrics.NewAppMetrics(a.log, frostfs.NewPoolStatistic(a.pool), a.cfg.GetBool(cfgPrometheusEnabled)) a.metrics.State().SetHealth(metrics.HealthStatusStarting) } func (a *App) initResolver() { var err error a.bucketResolver, err = resolver.NewBucketResolver(a.getResolverConfig()) if err != nil { a.log.Fatal(logs.FailedToCreateResolver, zap.Error(err)) } } func (a *App) getResolverConfig() ([]string, *resolver.Config) { resolveCfg := &resolver.Config{ FrostFS: frostfs.NewResolverFrostFS(a.pool), RPCAddress: a.cfg.GetString(cfgRPCEndpoint), } order := a.cfg.GetStringSlice(cfgResolveOrder) if resolveCfg.RPCAddress == "" { order = remove(order, resolver.NNSResolver) a.log.Warn(logs.ResolverNNSWontBeUsedSinceRPCEndpointIsntProvided) } if len(order) == 0 { a.log.Info(logs.ContainerResolverWillBeDisabled) } return order, resolveCfg } func (a *App) initTracing(ctx context.Context) { instanceID := "" if len(a.servers) > 0 { instanceID = a.servers[0].Address() } cfg := tracing.Config{ Enabled: a.cfg.GetBool(cfgTracingEnabled), Exporter: tracing.Exporter(a.cfg.GetString(cfgTracingExporter)), Endpoint: a.cfg.GetString(cfgTracingEndpoint), Service: "frostfs-s3-gw", InstanceID: instanceID, Version: version.Version, } updated, err := tracing.Setup(ctx, cfg) if err != nil { a.log.Warn(logs.FailedToInitializeTracing, zap.Error(err)) } if updated { a.log.Info(logs.TracingConfigUpdated) } } func (a *App) shutdownTracing() { const tracingShutdownTimeout = 5 * time.Second shdnCtx, cancel := context.WithTimeout(context.Background(), tracingShutdownTimeout) defer cancel() if err := tracing.Shutdown(shdnCtx); err != nil { a.log.Warn(logs.FailedToShutdownTracing, zap.Error(err)) } } func newMaxClients(cfg *viper.Viper) maxClientsConfig { config := maxClientsConfig{} config.count = fetchMaxClientsCount(cfg) config.deadline = fetchMaxClientsDeadline(cfg) return config } func getPools(ctx context.Context, logger *zap.Logger, cfg *viper.Viper) (*pool.Pool, *treepool.Pool, *keys.PrivateKey) { var prm pool.InitParameters var prmTree treepool.InitParameters password := wallet.GetPassword(cfg, cfgWalletPassphrase) key, err := wallet.GetKeyFromPath(cfg.GetString(cfgWalletPath), cfg.GetString(cfgWalletAddress), password) if err != nil { logger.Fatal(logs.CouldNotLoadFrostFSPrivateKey, zap.Error(err)) } prm.SetKey(&key.PrivateKey) prmTree.SetKey(key) logger.Info(logs.UsingCredentials, zap.String("FrostFS", hex.EncodeToString(key.PublicKey().Bytes()))) for _, peer := range fetchPeers(logger, cfg) { prm.AddNode(peer) prmTree.AddNode(peer) } connTimeout := fetchConnectTimeout(cfg) prm.SetNodeDialTimeout(connTimeout) prmTree.SetNodeDialTimeout(connTimeout) streamTimeout := fetchStreamTimeout(cfg) prm.SetNodeStreamTimeout(streamTimeout) prmTree.SetNodeStreamTimeout(streamTimeout) healthCheckTimeout := fetchHealthCheckTimeout(cfg) prm.SetHealthcheckTimeout(healthCheckTimeout) prmTree.SetHealthcheckTimeout(healthCheckTimeout) rebalanceInterval := fetchRebalanceInterval(cfg) prm.SetClientRebalanceInterval(rebalanceInterval) prmTree.SetClientRebalanceInterval(rebalanceInterval) errorThreshold := fetchErrorThreshold(cfg) prm.SetErrorThreshold(errorThreshold) prm.SetLogger(logger) prmTree.SetLogger(logger) prmTree.SetMaxRequestAttempts(cfg.GetInt(cfgTreePoolMaxAttempts)) var apiGRPCDialOpts []grpc.DialOption var treeGRPCDialOpts []grpc.DialOption if cfg.GetBool(cfgTracingEnabled) { interceptors := []grpc.DialOption{ grpc.WithUnaryInterceptor(grpctracing.NewUnaryClientInteceptor()), grpc.WithStreamInterceptor(grpctracing.NewStreamClientInterceptor()), } treeGRPCDialOpts = append(treeGRPCDialOpts, interceptors...) apiGRPCDialOpts = append(apiGRPCDialOpts, interceptors...) } prm.SetGRPCDialOptions(apiGRPCDialOpts...) prmTree.SetGRPCDialOptions(treeGRPCDialOpts...) p, err := pool.NewPool(prm) if err != nil { logger.Fatal(logs.FailedToCreateConnectionPool, zap.Error(err)) } if err = p.Dial(ctx); err != nil { logger.Fatal(logs.FailedToDialConnectionPool, zap.Error(err)) } treePool, err := treepool.NewPool(prmTree) if err != nil { logger.Fatal(logs.FailedToCreateTreePool, zap.Error(err)) } if err = treePool.Dial(ctx); err != nil { logger.Fatal(logs.FailedToDialTreePool, zap.Error(err)) } return p, treePool, key } func newPlacementPolicy(l *zap.Logger, v *viper.Viper) *placementPolicy { var policies placementPolicy policies.update(l, v) return &policies } func (p *placementPolicy) DefaultPlacementPolicy() netmap.PlacementPolicy { p.mu.RLock() defer p.mu.RUnlock() return p.defaultPolicy } func (p *placementPolicy) PlacementPolicy(name string) (netmap.PlacementPolicy, bool) { p.mu.RLock() policy, ok := p.regionMap[name] p.mu.RUnlock() return policy, ok } func (p *placementPolicy) CopiesNumbers(locationConstraint string) ([]uint32, bool) { p.mu.RLock() copiesNumbers, ok := p.copiesNumbers[locationConstraint] p.mu.RUnlock() return copiesNumbers, ok } func (p *placementPolicy) DefaultCopiesNumbers() []uint32 { p.mu.RLock() defer p.mu.RUnlock() return p.defaultCopiesNumbers } func (p *placementPolicy) update(l *zap.Logger, v *viper.Viper) { defaultPolicy := fetchDefaultPolicy(l, v) regionMap := fetchRegionMappingPolicies(l, v) defaultCopies := fetchDefaultCopiesNumbers(l, v) copiesNumbers := fetchCopiesNumbers(l, v) p.mu.Lock() defer p.mu.Unlock() p.defaultPolicy = defaultPolicy p.regionMap = regionMap p.defaultCopiesNumbers = defaultCopies p.copiesNumbers = copiesNumbers } func remove(list []string, element string) []string { for i, item := range list { if item == element { return append(list[:i], list[i+1:]...) } } return list } // Wait waits for an application to finish. // // Pre-logs a message about the launch of the application mentioning its // version (version.Version) and its name (frostfs-s3-gw). At the end, it writes // about the stop to the log. func (a *App) Wait() { a.log.Info(logs.ApplicationStarted, zap.String("name", "frostfs-s3-gw"), zap.String("version", version.Version), ) a.metrics.State().SetVersion(version.Version) a.setHealthStatus() <-a.webDone // wait for web-server to be stopped a.log.Info(logs.ApplicationFinished) } func (a *App) setHealthStatus() { a.metrics.State().SetHealth(metrics.HealthStatusReady) } // Serve runs HTTP server to handle S3 API requests. func (a *App) Serve(ctx context.Context) { // Attach S3 API: domains := a.cfg.GetStringSlice(cfgListenDomains) a.log.Info(logs.FetchDomainsPrepareToUseAPI, zap.Strings("domains", domains)) throttleOps := middleware.ThrottleOpts{ Limit: a.settings.maxClient.count, BacklogTimeout: a.settings.maxClient.deadline, } chiRouter := chi.NewRouter() api.AttachChi(chiRouter, domains, throttleOps, a.api, a.ctr, a.log, a.metrics) // Use mux.Router as http.Handler srv := new(http.Server) srv.Handler = chiRouter srv.ErrorLog = zap.NewStdLog(a.log) a.startServices() for i := range a.servers { go func(i int) { a.log.Info(logs.StartingServer, zap.String("address", a.servers[i].Address())) if err := srv.Serve(a.servers[i].Listener()); err != nil && err != http.ErrServerClosed { a.metrics.MarkUnhealthy(a.servers[i].Address()) a.log.Fatal(logs.ListenAndServe, zap.Error(err)) } }(i) } sigs := make(chan os.Signal, 1) signal.Notify(sigs, syscall.SIGHUP) LOOP: for { select { case <-ctx.Done(): break LOOP case <-sigs: a.configReload(ctx) } } ctx, cancel := shutdownContext() defer cancel() a.log.Info(logs.StoppingServer, zap.Error(srv.Shutdown(ctx))) a.metrics.Shutdown() a.stopServices() a.shutdownTracing() close(a.webDone) } func shutdownContext() (context.Context, context.CancelFunc) { return context.WithTimeout(context.Background(), defaultShutdownTimeout) } func (a *App) configReload(ctx context.Context) { a.log.Info(logs.SIGHUPConfigReloadStarted) if !a.cfg.IsSet(cmdConfig) && !a.cfg.IsSet(cmdConfigDir) { a.log.Warn(logs.FailedToReloadConfigBecauseItsMissed) return } if err := readInConfig(a.cfg); err != nil { a.log.Warn(logs.FailedToReloadConfig, zap.Error(err)) return } if err := a.bucketResolver.UpdateResolvers(a.getResolverConfig()); err != nil { a.log.Warn(logs.FailedToReloadResolvers, zap.Error(err)) } if err := a.updateServers(); err != nil { a.log.Warn(logs.FailedToReloadServerParameters, zap.Error(err)) } a.setRuntimeParameters() a.stopServices() a.startServices() a.updateSettings() a.metrics.SetEnabled(a.cfg.GetBool(cfgPrometheusEnabled)) a.initTracing(ctx) a.setHealthStatus() a.log.Info(logs.SIGHUPConfigReloadCompleted) } func (a *App) updateSettings() { if lvl, err := getLogLevel(a.cfg); err != nil { a.log.Warn(logs.LogLevelWontBeUpdated, zap.Error(err)) } else { a.settings.logLevel.SetLevel(lvl) } a.settings.policies.update(a.log, a.cfg) a.settings.xmlDecoder.UseDefaultNamespaceForCompleteMultipart(a.cfg.GetBool(cfgKludgeUseDefaultXMLNSForCompleteMultipartUpload)) a.settings.setBypassContentEncodingInChunks(a.cfg.GetBool(cfgKludgeBypassContentEncodingCheckInChunks)) a.settings.setClientCut(a.cfg.GetBool(cfgClientCut)) a.settings.setMD5Enabled(a.cfg.GetBool(cfgMD5Enabled)) } func (a *App) startServices() { a.services = a.services[:0] pprofService := NewPprofService(a.cfg, a.log) a.services = append(a.services, pprofService) go pprofService.Start() prometheusService := NewPrometheusService(a.cfg, a.log, a.metrics.Handler()) a.services = append(a.services, prometheusService) go prometheusService.Start() } func (a *App) initServers(ctx context.Context) { serversInfo := fetchServers(a.cfg) a.servers = make([]Server, 0, len(serversInfo)) for _, serverInfo := range serversInfo { fields := []zap.Field{ zap.String("address", serverInfo.Address), zap.Bool("tls enabled", serverInfo.TLS.Enabled), zap.String("tls cert", serverInfo.TLS.CertFile), zap.String("tls key", serverInfo.TLS.KeyFile), } srv, err := newServer(ctx, serverInfo) if err != nil { a.metrics.MarkUnhealthy(serverInfo.Address) a.log.Warn(logs.FailedToAddServer, append(fields, zap.Error(err))...) continue } a.metrics.MarkHealthy(serverInfo.Address) a.servers = append(a.servers, srv) a.log.Info(logs.AddServer, fields...) } if len(a.servers) == 0 { a.log.Fatal(logs.NoHealthyServers) } } func (a *App) updateServers() error { serversInfo := fetchServers(a.cfg) var found bool for _, serverInfo := range serversInfo { index := a.serverIndex(serverInfo.Address) if index == -1 { continue } if serverInfo.TLS.Enabled { if err := a.servers[index].UpdateCert(serverInfo.TLS.CertFile, serverInfo.TLS.KeyFile); err != nil { return fmt.Errorf("failed to update tls certs: %w", err) } } found = true } if !found { return fmt.Errorf("invalid servers configuration: no known server found") } return nil } func (a *App) serverIndex(address string) int { for i := range a.servers { if a.servers[i].Address() == address { return i } } return -1 } func (a *App) stopServices() { ctx, cancel := shutdownContext() defer cancel() for _, svc := range a.services { svc.ShutDown(ctx) } } func getNotificationsOptions(v *viper.Viper, l *zap.Logger) *notifications.Options { cfg := notifications.Options{} cfg.URL = v.GetString(cfgNATSEndpoint) cfg.Timeout = fetchNATSTimeout(v, l) cfg.TLSCertFilepath = v.GetString(cfgNATSTLSCertFile) cfg.TLSAuthPrivateKeyFilePath = v.GetString(cfgNATSAuthPrivateKeyFile) cfg.RootCAFiles = v.GetStringSlice(cfgNATSRootCAFiles) return &cfg } func getCacheOptions(v *viper.Viper, l *zap.Logger) *layer.CachesConfig { cacheCfg := layer.DefaultCachesConfigs(l) cacheCfg.Objects.Lifetime = fetchCacheLifetime(v, l, cfgObjectsCacheLifetime, cacheCfg.Objects.Lifetime) cacheCfg.Objects.Size = fetchCacheSize(v, l, cfgObjectsCacheSize, cacheCfg.Objects.Size) cacheCfg.ObjectsList.Lifetime = fetchCacheLifetime(v, l, cfgListObjectsCacheLifetime, cacheCfg.ObjectsList.Lifetime) cacheCfg.ObjectsList.Size = fetchCacheSize(v, l, cfgListObjectsCacheSize, cacheCfg.ObjectsList.Size) cacheCfg.Buckets.Lifetime = fetchCacheLifetime(v, l, cfgBucketsCacheLifetime, cacheCfg.Buckets.Lifetime) cacheCfg.Buckets.Size = fetchCacheSize(v, l, cfgBucketsCacheSize, cacheCfg.Buckets.Size) cacheCfg.Names.Lifetime = fetchCacheLifetime(v, l, cfgNamesCacheLifetime, cacheCfg.Names.Lifetime) cacheCfg.Names.Size = fetchCacheSize(v, l, cfgNamesCacheSize, cacheCfg.Names.Size) cacheCfg.System.Lifetime = fetchCacheLifetime(v, l, cfgSystemCacheLifetime, cacheCfg.System.Lifetime) cacheCfg.System.Size = fetchCacheSize(v, l, cfgSystemCacheSize, cacheCfg.System.Size) cacheCfg.AccessControl.Lifetime = fetchCacheLifetime(v, l, cfgAccessControlCacheLifetime, cacheCfg.AccessControl.Lifetime) cacheCfg.AccessControl.Size = fetchCacheSize(v, l, cfgAccessControlCacheSize, cacheCfg.AccessControl.Size) return cacheCfg } func getAccessBoxCacheConfig(v *viper.Viper, l *zap.Logger) *cache.Config { cacheCfg := cache.DefaultAccessBoxConfig(l) cacheCfg.Lifetime = fetchCacheLifetime(v, l, cfgAccessBoxCacheLifetime, cacheCfg.Lifetime) cacheCfg.Size = fetchCacheSize(v, l, cfgAccessBoxCacheSize, cacheCfg.Size) return cacheCfg } func (a *App) initHandler() { cfg := &handler.Config{ Policy: a.settings.policies, DefaultMaxAge: fetchDefaultMaxAge(a.cfg, a.log), NotificatorEnabled: a.cfg.GetBool(cfgEnableNATS), XMLDecoder: a.settings.xmlDecoder, } cfg.ResolveZoneList = a.cfg.GetStringSlice(cfgResolveBucketAllow) cfg.IsResolveListAllow = len(cfg.ResolveZoneList) > 0 if !cfg.IsResolveListAllow { cfg.ResolveZoneList = a.cfg.GetStringSlice(cfgResolveBucketDeny) } cfg.CompleteMultipartKeepalive = a.cfg.GetDuration(cfgKludgeCompleteMultipartUploadKeepalive) cfg.Kludge = a.settings cfg.Features = a.settings var err error a.api, err = handler.New(a.log, a.obj, a.nc, cfg) if err != nil { a.log.Fatal(logs.CouldNotInitializeAPIHandler, zap.Error(err)) } } func (a *App) setRuntimeParameters() { if len(os.Getenv("GOMEMLIMIT")) != 0 { // default limit < yaml limit < app env limit < GOMEMLIMIT a.log.Warn(logs.RuntimeSoftMemoryDefinedWithGOMEMLIMIT) return } softMemoryLimit := fetchSoftMemoryLimit(a.cfg) previous := debug.SetMemoryLimit(softMemoryLimit) if softMemoryLimit != previous { a.log.Info(logs.RuntimeSoftMemoryLimitUpdated, zap.Int64("new_value", softMemoryLimit), zap.Int64("old_value", previous)) } }