frostfs-node/pkg/local_object_storage/engine/control.go

package engine

import (
	"context"
	"errors"
	"fmt"
	"path/filepath"
	"strings"
	"sync"

	"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor"
	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard"
	"go.uber.org/zap"
	"golang.org/x/sync/errgroup"
)

type shardInitError struct {
	err error
	id  string
}

// Open opens all StorageEngine's components.
func (e *StorageEngine) Open(ctx context.Context) error {
	return e.open(ctx)
}

func (e *StorageEngine) open(ctx context.Context) error {
	e.mtx.Lock()
	defer e.mtx.Unlock()

	var wg sync.WaitGroup
	errCh := make(chan shardInitError, len(e.shards))

	for id, sh := range e.shards {
		wg.Add(1)
		go func(id string, sh *shard.Shard) {
			defer wg.Done()
			if err := sh.Open(ctx); err != nil {
				errCh <- shardInitError{
					err: err,
					id:  id,
				}
			}
		}(id, sh.Shard)
	}
	wg.Wait()
	close(errCh)

	for res := range errCh {
		if res.err != nil {
			e.log.Error(logs.EngineCouldNotOpenShardClosingAndSkipping,
				zap.String("id", res.id),
				zap.Error(res.err))

			sh := e.shards[res.id]
			delete(e.shards, res.id)

			err := sh.Close()
			if err != nil {
				e.log.Error(logs.EngineCouldNotClosePartiallyInitializedShard,
					zap.String("id", res.id),
					zap.Error(res.err))
			}

			continue
		}
	}

	return nil
}

// Init initializes all StorageEngine's components.
func (e *StorageEngine) Init(ctx context.Context) error {
	e.mtx.Lock()
	defer e.mtx.Unlock()

	errCh := make(chan shardInitError, len(e.shards))
	var eg errgroup.Group
	if e.cfg.lowMem && e.anyShardRequiresRefill() {
		eg.SetLimit(1)
	}

	for id, sh := range e.shards {
		id := id
		sh := sh
		eg.Go(func() error {
			if err := sh.Init(ctx); err != nil {
				errCh <- shardInitError{
					err: err,
					id:  id,
				}
			}
			return nil
		})
	}
	err := eg.Wait()
	close(errCh)
	if err != nil {
		return fmt.Errorf("failed to initialize shards: %w", err)
	}

	for res := range errCh {
		if res.err != nil {
			if errors.Is(res.err, blobstor.ErrInitBlobovniczas) {
				e.log.Error(logs.EngineCouldNotInitializeShardClosingAndSkipping,
					zap.String("id", res.id),
					zap.Error(res.err))

				sh := e.shards[res.id]
				delete(e.shards, res.id)

				err := sh.Close()
				if err != nil {
					e.log.Error(logs.EngineCouldNotClosePartiallyInitializedShard,
						zap.String("id", res.id),
						zap.Error(res.err))
				}

				continue
			}
			return fmt.Errorf("could not initialize shard %s: %w", res.id, res.err)
		}
	}

	if len(e.shards) == 0 {
		return errors.New("failed initialization on all shards")
	}

	e.wg.Add(1)
	go e.setModeLoop()

	return nil
}

func (e *StorageEngine) anyShardRequiresRefill() bool {
	for _, sh := range e.shards {
		if sh.NeedRefillMetabase() {
			return true
		}
	}
	return false
}

var errClosed = errors.New("storage engine is closed")

// Close releases all StorageEngine's components. Waits for all data-related operations to complete.
// After the call, all the next ones will fail.
//
// The method MUST only be called when the application exits.
func (e *StorageEngine) Close(ctx context.Context) error {
	close(e.closeCh)
	defer e.wg.Wait()
	return e.setBlockExecErr(ctx, errClosed)
}

// closes all shards. Never returns an error, shard errors are logged.
func (e *StorageEngine) close(releasePools bool) error {
	e.mtx.RLock()
	defer e.mtx.RUnlock()

	if releasePools {
		for _, p := range e.shardPools {
			p.Release()
		}
	}

	for id, sh := range e.shards {
		if err := sh.Close(); err != nil {
			e.log.Debug(logs.EngineCouldNotCloseShard,
				zap.String("id", id),
				zap.String("error", err.Error()),
			)
		}
	}

	return nil
}

// executes op if execution is not blocked, otherwise returns blocking error.
//
// Can be called concurrently with setBlockExecErr.
func (e *StorageEngine) execIfNotBlocked(op func() error) error {
	e.blockExec.mtx.RLock()
	defer e.blockExec.mtx.RUnlock()

	if e.blockExec.err != nil {
		return e.blockExec.err
	}

	return op()
}

// sets the flag of blocking execution of all data operations according to err:
//   - err != nil, then blocks the execution. If exec wasn't blocked, calls close method
//     (if err == errClosed => additionally releases pools and does not allow to resume executions).
//   - otherwise, resumes execution. If exec was blocked, calls open method.
//
// Can be called concurrently with exec. In this case it waits for all executions to complete.
func (e *StorageEngine) setBlockExecErr(ctx context.Context, err error) error {
	e.blockExec.mtx.Lock()
	defer e.blockExec.mtx.Unlock()

	prevErr := e.blockExec.err

	wasClosed := errors.Is(prevErr, errClosed)
	if wasClosed {
		return errClosed
	}

	e.blockExec.err = err

	if err == nil {
		if prevErr != nil { // block -> ok
			return e.open(ctx)
		}
	} else if prevErr == nil { // ok -> block
		return e.close(errors.Is(err, errClosed))
	}

	// otherwise do nothing

	return nil
}

// BlockExecution blocks the execution of any data-related operation. All blocked ops will return err.
// To resume the execution, use ResumeExecution method.
//
// Сan be called regardless of the fact of the previous blocking. If execution wasn't blocked, releases all resources
// similar to Close. Can be called concurrently with Close and any data related method (waits for all executions
// to complete). Returns error if any Close has been called before.
//
// Must not be called concurrently with either Open or Init.
//
// Note: technically passing nil error will resume the execution, otherwise, it is recommended to call ResumeExecution
// for this.
func (e *StorageEngine) BlockExecution(err error) error {
	return e.setBlockExecErr(context.Background(), err)
}

// ResumeExecution resumes the execution of any data-related operation.
// To block the execution, use BlockExecution method.
//
// Сan be called regardless of the fact of the previous blocking. If execution was blocked, prepares all resources
// similar to Open. Can be called concurrently with Close and any data related method (waits for all executions
// to complete). Returns error if any Close has been called before.
//
// Must not be called concurrently with either Open or Init.
func (e *StorageEngine) ResumeExecution() error {
	return e.setBlockExecErr(context.Background(), nil)
}

type ReConfiguration struct {
	errorsThreshold uint32
	shardPoolSize   uint32

	shards map[string][]shard.Option // meta path -> shard opts
}

// SetErrorsThreshold sets a size amount of errors after which
// shard is moved to read-only mode.
func (rCfg *ReConfiguration) SetErrorsThreshold(errorsThreshold uint32) {
	rCfg.errorsThreshold = errorsThreshold
}

// SetShardPoolSize sets a size of worker pool for each shard.
func (rCfg *ReConfiguration) SetShardPoolSize(shardPoolSize uint32) {
	rCfg.shardPoolSize = shardPoolSize
}

// AddShard adds a shard for the reconfiguration.
// Shard identifier is calculated from paths used in blobstor.
func (rCfg *ReConfiguration) AddShard(id string, opts []shard.Option) {
	if rCfg.shards == nil {
		rCfg.shards = make(map[string][]shard.Option)
	}

	if _, found := rCfg.shards[id]; found {
		return
	}

	rCfg.shards[id] = opts
}

// Reload reloads StorageEngine's configuration in runtime.
func (e *StorageEngine) Reload(ctx context.Context, rcfg ReConfiguration) error {
	type reloadInfo struct {
		sh   *shard.Shard
		opts []shard.Option
	}

	e.mtx.RLock()

	var shardsToRemove []string // shards IDs
	var shardsToAdd []string    // shard config identifiers (blobstor paths concatenation)
	var shardsToReload []reloadInfo

	// mark removed shards for removal
	for id, sh := range e.shards {
		_, ok := rcfg.shards[calculateShardID(sh.DumpInfo())]
		if !ok {
			shardsToRemove = append(shardsToRemove, id)
		}
	}

loop:
	for newID := range rcfg.shards {
		for _, sh := range e.shards {
			// This calculation should be kept in sync with node
			// configuration parsing during SIGHUP.
			if newID == calculateShardID(sh.DumpInfo()) {
				shardsToReload = append(shardsToReload, reloadInfo{
					sh:   sh.Shard,
					opts: rcfg.shards[newID],
				})
				continue loop
			}
		}

		shardsToAdd = append(shardsToAdd, newID)
	}

	e.mtx.RUnlock()

	e.removeShards(shardsToRemove...)

	for _, p := range shardsToReload {
		err := p.sh.Reload(ctx, p.opts...)
		if err != nil {
			e.log.Error(logs.EngineCouldNotReloadAShard,
				zap.Stringer("shard id", p.sh.ID()),
				zap.Error(err))
		}
	}

	for _, newID := range shardsToAdd {
		sh, err := e.createShard(ctx, rcfg.shards[newID])
		if err != nil {
			return fmt.Errorf("could not add new shard with '%s' metabase path: %w", newID, err)
		}

		idStr := sh.ID().String()

		err = sh.Open(ctx)
		if err == nil {
			err = sh.Init(ctx)
		}
		if err != nil {
			_ = sh.Close()
			return fmt.Errorf("could not init %s shard: %w", idStr, err)
		}

		err = e.addShard(sh)
		if err != nil {
			_ = sh.Close()
			return fmt.Errorf("could not add %s shard: %w", idStr, err)
		}

		e.log.Info(logs.EngineAddedNewShard, zap.String("id", idStr))
	}

	return nil
}

func calculateShardID(info shard.Info) string {
	// This calculation should be kept in sync with node
	// configuration parsing during SIGHUP.
	var sb strings.Builder
	for _, sub := range info.BlobStorInfo.SubStorages {
		sb.WriteString(filepath.Clean(sub.Path))
	}
	return sb.String()
}