[TrueCloudLab#12] Require only one healthy server

Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>

cmd/s3-gw/app.go

@@ -527,37 +527,61 @@ func (a *App) startServices() {
 func (a *App) initServers(ctx context.Context) {
 	serversInfo := fetchServers(a.cfg)
 
-	a.servers = make([]Server, len(serversInfo))
-	for i, serverInfo := range serversInfo {
-		a.log.Info("added server",
+	a.servers = make([]Server, 0, len(serversInfo))
+	for _, serverInfo := range serversInfo {
+		fields := []zap.Field{
 			zap.String("address", serverInfo.Address), zap.Bool("tls enabled", serverInfo.TLS.Enabled),
-			zap.String("tls cert", serverInfo.TLS.CertFile), zap.String("tls key", serverInfo.TLS.KeyFile))
-		a.servers[i] = newServer(ctx, serverInfo, a.log)
+			zap.String("tls cert", serverInfo.TLS.CertFile), zap.String("tls key", serverInfo.TLS.KeyFile),
+		}
+		srv, err := newServer(ctx, serverInfo)
+		if err != nil {
+			a.log.Warn("failed to add server", append(fields, zap.Error(err))...)
+			continue
+		}
+
+		a.servers = append(a.servers, srv)
+		a.log.Info("add server", fields...)
+	}
+
+	if len(a.servers) == 0 {
+		a.log.Fatal("no healthy servers")
 	}
 }
 
 func (a *App) updateServers() error {
 	serversInfo := fetchServers(a.cfg)
 
-	if len(serversInfo) != len(a.servers) {
-		return fmt.Errorf("invalid servers configuration: amount mismatch: old '%d', new '%d", len(a.servers), len(serversInfo))
-	}
-
-	for i, serverInfo := range serversInfo {
-		if serverInfo.Address != a.servers[i].Address() {
-			return fmt.Errorf("invalid servers configuration: addresses mismatch: old '%s', new '%s", a.servers[i].Address(), serverInfo.Address)
+	var found bool
+	for _, serverInfo := range serversInfo {
+		index := a.serverIndex(serverInfo.Address)
+		if index == -1 {
+			continue
 		}
 
 		if serverInfo.TLS.Enabled {
-			if err := a.servers[i].UpdateCert(serverInfo.TLS.CertFile, serverInfo.TLS.KeyFile); err != nil {
+			if err := a.servers[index].UpdateCert(serverInfo.TLS.CertFile, serverInfo.TLS.KeyFile); err != nil {
 				return fmt.Errorf("failed to update tls certs: %w", err)
 			}
 		}
+		found = true
+	}
+
+	if !found {
+		return fmt.Errorf("invalid servers configuration: no known server found")
 	}
 
 	return nil
 }
 
+func (a *App) serverIndex(address string) int {
+	for i := range a.servers {
+		if a.servers[i].Address() == address {
+			return i
+		}
+	}
+	return -1
+}
+
 func (a *App) stopServices() {
 	ctx, cancel := shutdownContext()
 	defer cancel()

cmd/s3-gw/server.go

@@ -7,8 +7,6 @@ import (
 	"fmt"
 	"net"
 	"sync"
-
-	"go.uber.org/zap"
 )
 
 type (
@@ -57,11 +55,11 @@ func (s *server) UpdateCert(certFile, keyFile string) error {
 	return s.tlsProvider.UpdateCert(certFile, keyFile)
 }
 
-func newServer(ctx context.Context, serverInfo ServerInfo, logger *zap.Logger) *server {
+func newServer(ctx context.Context, serverInfo ServerInfo) (*server, error) {
 	var lic net.ListenConfig
 	ln, err := lic.Listen(ctx, "tcp", serverInfo.Address)
 	if err != nil {
-		logger.Fatal("could not prepare listener", zap.String("address", serverInfo.Address), zap.Error(err))
+		return nil, fmt.Errorf("could not prepare listener: %w", err)
 	}
 
 	tlsProvider := &certProvider{
@@ -70,7 +68,7 @@ func newServer(ctx context.Context, serverInfo ServerInfo, logger *zap.Logger) *
 
 	if serverInfo.TLS.Enabled {
 		if err = tlsProvider.UpdateCert(serverInfo.TLS.CertFile, serverInfo.TLS.KeyFile); err != nil {
-			logger.Fatal("failed to update cert", zap.Error(err))
+			return nil, fmt.Errorf("failed to update cert: %w", err)
 		}
 
 		ln = tls.NewListener(ln, &tls.Config{
@@ -82,7 +80,7 @@ func newServer(ctx context.Context, serverInfo ServerInfo, logger *zap.Logger) *
 		address:     serverInfo.Address,
 		listener:    ln,
 		tlsProvider: tlsProvider,
-	}
+	}, nil
 }
 
 func (p *certProvider) GetCertificate(*tls.ClientHelloInfo) (*tls.Certificate, error) {