frostfs-s3-gw/cmd/s3-gw/app.go

package main

import (
	"context"
	"crypto/tls"
	"encoding/hex"
	"errors"
	"fmt"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strconv"
	"sync"
	"syscall"
	"time"

	"github.com/gorilla/mux"
	"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
	"github.com/nspcc-dev/neofs-s3-gw/api"
	"github.com/nspcc-dev/neofs-s3-gw/api/auth"
	"github.com/nspcc-dev/neofs-s3-gw/api/cache"
	"github.com/nspcc-dev/neofs-s3-gw/api/handler"
	"github.com/nspcc-dev/neofs-s3-gw/api/layer"
	"github.com/nspcc-dev/neofs-s3-gw/api/notifications"
	"github.com/nspcc-dev/neofs-s3-gw/api/resolver"
	"github.com/nspcc-dev/neofs-s3-gw/internal/neofs"
	"github.com/nspcc-dev/neofs-s3-gw/internal/version"
	"github.com/nspcc-dev/neofs-s3-gw/internal/wallet"
	"github.com/nspcc-dev/neofs-sdk-go/pool"
	"github.com/spf13/viper"
	"go.uber.org/zap"
)

type (
	// App is the main application structure.
	App struct {
		ctr  auth.Center
		log  *zap.Logger
		cfg  *viper.Viper
		pool *pool.Pool
		obj  layer.Client
		api  api.Handler

		metrics        *appMetrics
		bucketResolver *resolver.BucketResolver
		tlsProvider    *certProvider

		maxClients api.MaxClients

		webDone chan struct{}
		wrkDone chan struct{}

		services []*Service
		settings *appSettings
	}

	appSettings struct {
		LogLevel zap.AtomicLevel
	}

	Logger struct {
		logger *zap.Logger
		lvl    zap.AtomicLevel
	}

	certProvider struct {
		Enabled bool

		mu       sync.RWMutex
		certPath string
		keyPath  string
		cert     *tls.Certificate
	}

	appMetrics struct {
		logger   *zap.Logger
		provider GateMetricsCollector
		mu       sync.RWMutex
		enabled  bool
	}

	GateMetricsCollector interface {
		SetHealth(int32)
		Unregister()
	}
)

func newApp(ctx context.Context, log *Logger, v *viper.Viper) *App {
	l := log.logger

	var (
		key    *keys.PrivateKey
		err    error
		caller api.Handler
		ctr    auth.Center
		obj    layer.Client
		nc     *notifications.Controller

		prmPool pool.InitParameters

		reBalance  = defaultRebalanceInterval
		conTimeout = defaultConnectTimeout
		hckTimeout = defaultHealthcheckTimeout

		maxClientsCount    = defaultMaxClientsCount
		maxClientsDeadline = defaultMaxClientsDeadline
		poolErrorThreshold = defaultPoolErrorThreshold
	)

	if v := v.GetDuration(cfgConnectTimeout); v > 0 {
		conTimeout = v
	}

	if v := v.GetDuration(cfgHealthcheckTimeout); v > 0 {
		hckTimeout = v
	}

	if v := v.GetInt(cfgMaxClientsCount); v > 0 {
		maxClientsCount = v
	}

	if v := v.GetDuration(cfgMaxClientsDeadline); v > 0 {
		maxClientsDeadline = v
	}

	if v := v.GetDuration(cfgRebalanceInterval); v > 0 {
		reBalance = v
	}

	if v := v.GetUint32(cfgPoolErrorThreshold); v > 0 {
		poolErrorThreshold = v
	}

	password := wallet.GetPassword(v, cfgWalletPassphrase)
	if key, err = wallet.GetKeyFromPath(v.GetString(cfgWalletPath), v.GetString(cfgWalletAddress), password); err != nil {
		l.Fatal("could not load NeoFS private key", zap.Error(err))
	}

	l.Info("using credentials", zap.String("NeoFS", hex.EncodeToString(key.PublicKey().Bytes())))

	prmPool.SetKey(&key.PrivateKey)
	prmPool.SetNodeDialTimeout(conTimeout)
	prmPool.SetHealthcheckTimeout(hckTimeout)
	prmPool.SetErrorThreshold(poolErrorThreshold)
	prmPool.SetClientRebalanceInterval(reBalance)
	for _, peer := range fetchPeers(l, v) {
		prmPool.AddNode(peer)
	}

	conns, err := pool.NewPool(prmPool)
	if err != nil {
		l.Fatal("failed to create connection pool", zap.Error(err))
	}

	if err = conns.Dial(ctx); err != nil {
		l.Fatal("failed to dial connection pool", zap.Error(err))
	}

	// prepare random key for anonymous requests
	randomKey, err := keys.NewPrivateKey()
	if err != nil {
		l.Fatal("couldn't generate random key", zap.Error(err))
	}

	resolveCfg := &resolver.Config{
		NeoFS:      neofs.NewResolverNeoFS(conns),
		RPCAddress: v.GetString(cfgRPCEndpoint),
	}

	order := v.GetStringSlice(cfgResolveOrder)
	if resolveCfg.RPCAddress == "" {
		order = remove(order, resolver.NNSResolver)
		l.Warn(fmt.Sprintf("resolver '%s' won't be used since '%s' isn't provided", resolver.NNSResolver, cfgRPCEndpoint))
	}

	bucketResolver, err := resolver.NewBucketResolver(order, resolveCfg)
	if err != nil {
		l.Fatal("failed to form resolver", zap.Error(err))
	}

	treeServiceEndpoint := v.GetString(cfgTreeServiceEndpoint)
	treeService, err := neofs.NewTreeClient(treeServiceEndpoint, key)
	if err != nil {
		l.Fatal("failed to create tree service", zap.Error(err))
	}
	l.Info("init tree service", zap.String("endpoint", treeServiceEndpoint))

	layerCfg := &layer.Config{
		Caches: getCacheOptions(v, l),
		AnonKey: layer.AnonymousKey{
			Key: randomKey,
		},
		Resolver:    bucketResolver,
		TreeService: treeService,
	}

	// prepare object layer
	obj = layer.NewLayer(l, neofs.NewNeoFS(conns), layerCfg)

	if v.GetBool(cfgEnableNATS) {
		nopts := getNotificationsOptions(v, l)
		nc, err = notifications.NewController(nopts, l)
		if err != nil {
			l.Fatal("failed to enable notifications", zap.Error(err))
		}

		if err = obj.Initialize(ctx, nc); err != nil {
			l.Fatal("couldn't initialize layer", zap.Error(err))
		}
	}

	// prepare auth center
	ctr = auth.New(neofs.NewAuthmateNeoFS(conns), key, v.GetStringSlice(cfgAllowedAccessKeyIDPrefixes), getAccessBoxCacheConfig(v, l))
	handlerOptions := getHandlerOptions(v, l)

	if caller, err = handler.New(l, obj, nc, handlerOptions); err != nil {
		l.Fatal("could not initialize API handler", zap.Error(err))
	}

	app := &App{
		ctr:  ctr,
		log:  l,
		cfg:  v,
		pool: conns,
		obj:  obj,
		api:  caller,

		webDone: make(chan struct{}, 1),
		wrkDone: make(chan struct{}, 1),

		maxClients: api.NewMaxClientsMiddleware(maxClientsCount, maxClientsDeadline),
	}

	app.initMetrics()
	app.initResolver()
	app.initTLSProvider()

	return app
}

func (a *App) initMetrics() {
	gateMetricsProvider := newGateMetrics(neofs.NewPoolStatistic(a.pool))
	a.metrics = newAppMetrics(a.log, gateMetricsProvider, a.cfg.GetBool(cfgPrometheusEnabled))
}

func (a *App) initResolver() {
	var err error
	a.bucketResolver, err = resolver.NewBucketResolver(a.getResolverConfig())
	if err != nil {
		a.log.Fatal("failed to create resolver", zap.Error(err))
	}
}

func (a *App) initTLSProvider() {
	a.tlsProvider = &certProvider{
		Enabled: a.cfg.IsSet(cfgTLSCertFile) || a.cfg.IsSet(cfgTLSKeyFile),
	}
}

func (a *App) getResolverConfig() ([]string, *resolver.Config) {
	resolveCfg := &resolver.Config{
		NeoFS:      neofs.NewResolverNeoFS(a.pool),
		RPCAddress: a.cfg.GetString(cfgRPCEndpoint),
	}

	order := a.cfg.GetStringSlice(cfgResolveOrder)
	if resolveCfg.RPCAddress == "" {
		order = remove(order, resolver.NNSResolver)
		a.log.Warn(fmt.Sprintf("resolver '%s' won't be used since '%s' isn't provided", resolver.NNSResolver, cfgRPCEndpoint))
	}

	if len(order) == 0 {
		a.log.Info("container resolver will be disabled because of resolvers 'resolver_order' is empty")
	}

	return order, resolveCfg
}

func newAppMetrics(logger *zap.Logger, provider GateMetricsCollector, enabled bool) *appMetrics {
	if !enabled {
		logger.Warn("metrics are disabled")
	}
	return &appMetrics{
		logger:   logger,
		provider: provider,
	}
}

func (m *appMetrics) SetEnabled(enabled bool) {
	if !enabled {
		m.logger.Warn("metrics are disabled")
	}

	m.mu.Lock()
	m.enabled = enabled
	m.mu.Unlock()
}

func (m *appMetrics) SetHealth(status int32) {
	m.mu.RLock()
	if !m.enabled {
		m.mu.RUnlock()
		return
	}
	m.mu.RUnlock()

	m.provider.SetHealth(status)
}

func (m *appMetrics) Shutdown() {
	m.mu.Lock()
	if m.enabled {
		m.provider.SetHealth(0)
		m.enabled = false
	}
	m.provider.Unregister()
	m.mu.Unlock()
}

func (p *certProvider) GetCertificate(*tls.ClientHelloInfo) (*tls.Certificate, error) {
	if !p.Enabled {
		return nil, errors.New("cert provider: disabled")
	}

	p.mu.RLock()
	defer p.mu.RUnlock()
	return p.cert, nil
}

func (p *certProvider) UpdateCert(certPath, keyPath string) error {
	if !p.Enabled {
		return fmt.Errorf("tls disabled")
	}

	cert, err := tls.LoadX509KeyPair(certPath, keyPath)
	if err != nil {
		return fmt.Errorf("cannot load TLS key pair from certFile '%s' and keyFile '%s': %w", certPath, keyPath, err)
	}

	p.mu.Lock()
	p.certPath = certPath
	p.keyPath = keyPath
	p.cert = &cert
	p.mu.Unlock()
	return nil
}

func (p *certProvider) FilePaths() (string, string) {
	if !p.Enabled {
		return "", ""
	}

	p.mu.RLock()
	defer p.mu.RUnlock()
	return p.certPath, p.keyPath
}

func remove(list []string, element string) []string {
	for i, item := range list {
		if item == element {
			return append(list[:i], list[i+1:]...)
		}
	}
	return list
}

// Wait waits for an application to finish.
//
// Pre-logs a message about the launch of the application mentioning its
// version (version.Version) and its name (neofs-s3-gw). At the end, it writes
// about the stop to the log.
func (a *App) Wait() {
	a.log.Info("application started",
		zap.String("name", "neofs-s3-gw"),
		zap.String("version", version.Version),
	)

	a.setHealthStatus()

	<-a.webDone // wait for web-server to be stopped

	a.log.Info("application finished")
}

func (a *App) setHealthStatus() {
	a.metrics.SetHealth(1)
}

// Serve runs HTTP server to handle S3 API requests.
func (a *App) Serve(ctx context.Context) {
	// Attach S3 API:
	domains := a.cfg.GetStringSlice(cfgListenDomains)
	a.log.Info("fetch domains, prepare to use API", zap.Strings("domains", domains))
	router := mux.NewRouter().SkipClean(true).UseEncodedPath()
	api.Attach(router, domains, a.maxClients, a.api, a.ctr, a.log)

	// Use mux.Router as http.Handler
	srv := new(http.Server)
	srv.Handler = router
	srv.ErrorLog = zap.NewStdLog(a.log)

	a.startServices()

	go func() {
		addr := a.cfg.GetString(cfgListenAddress)
		a.log.Info("starting server", zap.String("bind", addr))

		var lic net.ListenConfig
		ln, err := lic.Listen(ctx, "tcp", addr)
		if err != nil {
			a.log.Fatal("could not prepare listener", zap.Error(err))
		}

		if a.tlsProvider.Enabled {
			certFile := a.cfg.GetString(cfgTLSCertFile)
			keyFile := a.cfg.GetString(cfgTLSKeyFile)

			a.log.Info("using certificate", zap.String("cert", certFile), zap.String("key", keyFile))
			if err = a.tlsProvider.UpdateCert(certFile, keyFile); err != nil {
				a.log.Fatal("failed to update cert", zap.Error(err))
			}

			lnTLS := tls.NewListener(ln, &tls.Config{
				GetCertificate: a.tlsProvider.GetCertificate,
			})

			if err = srv.ServeTLS(lnTLS, certFile, keyFile); err != nil && err != http.ErrServerClosed {
				a.log.Fatal("listen and serve", zap.Error(err))
			}
		} else {
			if err = srv.Serve(ln); err != nil && err != http.ErrServerClosed {
				a.log.Fatal("listen and serve", zap.Error(err))
			}
		}
	}()

	sigs := make(chan os.Signal, 1)
	signal.Notify(sigs, syscall.SIGHUP)

LOOP:
	for {
		select {
		case <-ctx.Done():
			break LOOP
		case <-sigs:
			a.configReload()
		}
	}

	ctx, cancel := shutdownContext()
	defer cancel()

	a.log.Info("stopping server", zap.Error(srv.Shutdown(ctx)))

	a.metrics.Shutdown()
	a.stopServices()

	close(a.webDone)
}

func shutdownContext() (context.Context, context.CancelFunc) {
	return context.WithTimeout(context.Background(), defaultShutdownTimeout)
}

func (a *App) configReload() {
	a.log.Info("SIGHUP config reload started")

	if !a.cfg.IsSet(cmdConfig) {
		a.log.Warn("failed to reload config because it's missed")
		return
	}
	if err := readConfig(a.cfg); err != nil {
		a.log.Warn("failed to reload config", zap.Error(err))
		return
	}

	if err := a.bucketResolver.UpdateResolvers(a.getResolverConfig()); err != nil {
		a.log.Warn("failed to reload resolvers", zap.Error(err))
	}

	if err := a.tlsProvider.UpdateCert(a.cfg.GetString(cfgTLSCertFile), a.cfg.GetString(cfgTLSKeyFile)); err != nil {
		a.log.Warn("failed to reload TLS certs", zap.Error(err))
	}

	a.stopServices()
	a.startServices()

	a.updateSettings()

	a.metrics.SetEnabled(a.cfg.GetBool(cfgPrometheusEnabled))
	a.setHealthStatus()

	a.log.Info("SIGHUP config reload completed")
}

func (a *App) updateSettings() {
	if lvl, err := getLogLevel(a.cfg); err != nil {
		a.log.Warn("log level won't be updated", zap.Error(err))
	} else {
		a.settings.LogLevel.SetLevel(lvl)
	}
}

func (a *App) startServices() {
	pprofService := NewPprofService(a.cfg, a.log)
	a.services = append(a.services, pprofService)
	go pprofService.Start()

	prometheusService := NewPrometheusService(a.cfg, a.log)
	a.services = append(a.services, prometheusService)
	go prometheusService.Start()
}

func (a *App) stopServices() {
	ctx, cancel := shutdownContext()
	defer cancel()

	for _, svc := range a.services {
		svc.ShutDown(ctx)
	}
}

func getNotificationsOptions(v *viper.Viper, l *zap.Logger) *notifications.Options {
	cfg := notifications.Options{}
	cfg.URL = v.GetString(cfgNATSEndpoint)
	cfg.Timeout = v.GetDuration(cfgNATSTimeout)
	if cfg.Timeout <= 0 {
		l.Error("invalid lifetime, using default value (in seconds)",
			zap.String("parameter", cfgNATSTimeout),
			zap.Duration("value in config", cfg.Timeout),
			zap.Duration("default", notifications.DefaultTimeout))
		cfg.Timeout = notifications.DefaultTimeout
	}
	cfg.TLSCertFilepath = v.GetString(cfgNATSTLSCertFile)
	cfg.TLSAuthPrivateKeyFilePath = v.GetString(cfgNATSAuthPrivateKeyFile)
	cfg.RootCAFiles = v.GetStringSlice(cfgNATSRootCAFiles)

	return &cfg
}

func getCacheOptions(v *viper.Viper, l *zap.Logger) *layer.CachesConfig {
	cacheCfg := layer.DefaultCachesConfigs(l)

	cacheCfg.Objects.Lifetime = getLifetime(v, l, cfgObjectsCacheLifetime, cacheCfg.Objects.Lifetime)
	cacheCfg.Objects.Size = getSize(v, l, cfgObjectsCacheSize, cacheCfg.Objects.Size)

	cacheCfg.ObjectsList.Lifetime = getLifetime(v, l, cfgListObjectsCacheLifetime, cacheCfg.ObjectsList.Lifetime)
	cacheCfg.ObjectsList.Size = getSize(v, l, cfgListObjectsCacheSize, cacheCfg.ObjectsList.Size)

	cacheCfg.Buckets.Lifetime = getLifetime(v, l, cfgBucketsCacheLifetime, cacheCfg.Buckets.Lifetime)
	cacheCfg.Buckets.Size = getSize(v, l, cfgBucketsCacheSize, cacheCfg.Buckets.Size)

	cacheCfg.Names.Lifetime = getLifetime(v, l, cfgNamesCacheLifetime, cacheCfg.Names.Lifetime)
	cacheCfg.Names.Size = getSize(v, l, cfgNamesCacheSize, cacheCfg.Names.Size)

	cacheCfg.System.Lifetime = getLifetime(v, l, cfgSystemLifetimeSize, cacheCfg.System.Lifetime)
	cacheCfg.System.Size = getSize(v, l, cfgSystemCacheSize, cacheCfg.System.Size)

	return cacheCfg
}

func getLifetime(v *viper.Viper, l *zap.Logger, cfgEntry string, defaultValue time.Duration) time.Duration {
	if v.IsSet(cfgEntry) {
		lifetime := v.GetDuration(cfgEntry)
		if lifetime <= 0 {
			l.Error("invalid lifetime, using default value (in seconds)",
				zap.String("parameter", cfgEntry),
				zap.Duration("value in config", lifetime),
				zap.Duration("default", defaultValue))
		} else {
			return lifetime
		}
	}
	return defaultValue
}

func getSize(v *viper.Viper, l *zap.Logger, cfgEntry string, defaultValue int) int {
	if v.IsSet(cfgEntry) {
		size := v.GetInt(cfgEntry)
		if size <= 0 {
			l.Error("invalid cache size, using default value",
				zap.String("parameter", cfgEntry),
				zap.Int("value in config", size),
				zap.Int("default", defaultValue))
		} else {
			return size
		}
	}
	return defaultValue
}

func getAccessBoxCacheConfig(v *viper.Viper, l *zap.Logger) *cache.Config {
	cacheCfg := cache.DefaultAccessBoxConfig(l)

	cacheCfg.Lifetime = getLifetime(v, l, cfgAccessBoxCacheLifetime, cacheCfg.Lifetime)
	cacheCfg.Size = getSize(v, l, cfgAccessBoxCacheSize, cacheCfg.Size)

	return cacheCfg
}

func getHandlerOptions(v *viper.Viper, l *zap.Logger) *handler.Config {
	var (
		cfg             handler.Config
		err             error
		policyStr       = handler.DefaultPolicy
		defaultMaxAge   = handler.DefaultMaxAge
		setCopiesNumber = handler.DefaultCopiesNumber
	)

	if v.IsSet(cfgDefaultPolicy) {
		policyStr = v.GetString(cfgDefaultPolicy)
	}

	if err = cfg.DefaultPolicy.DecodeString(policyStr); err != nil {
		l.Fatal("couldn't parse container default policy",
			zap.Error(err))
	}

	if v.IsSet(cfgDefaultMaxAge) {
		defaultMaxAge = v.GetInt(cfgDefaultMaxAge)

		if defaultMaxAge <= 0 && defaultMaxAge != -1 {
			l.Fatal("invalid defaultMaxAge",
				zap.String("parameter", cfgDefaultMaxAge),
				zap.String("value in config", strconv.Itoa(defaultMaxAge)))
		}
	}

	if val := v.GetUint32(cfgSetCopiesNumber); val > 0 {
		setCopiesNumber = val
	}

	cfg.DefaultMaxAge = defaultMaxAge
	cfg.NotificatorEnabled = v.GetBool(cfgEnableNATS)
	cfg.TLSEnabled = v.IsSet(cfgTLSKeyFile) && v.IsSet(cfgTLSCertFile)
	cfg.CopiesNumber = setCopiesNumber

	return &cfg
}