frostfs-node/pkg/services/object/common/writer/common.go

package writer

import (
	"context"
	"fmt"
	"sync"
	"sync/atomic"

	"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
	svcutil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util"
	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement"
	objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
	"go.uber.org/zap"
)

type NodeIterator struct {
	Traversal
	cfg *Config
}

func (c *Config) NewNodeIterator(opts []placement.Option) *NodeIterator {
	return &NodeIterator{
		Traversal: Traversal{
			Opts:    opts,
			Exclude: make(map[string]*bool),
		},
		cfg: c,
	}
}

func (n *NodeIterator) ForEachNode(ctx context.Context, f func(context.Context, NodeDescriptor) error) error {
	traverser, err := placement.NewTraverser(n.Traversal.Opts...)
	if err != nil {
		return fmt.Errorf("could not create object placement traverser: %w", err)
	}

	resErr := &atomic.Value{}

	// Must iterate over all replicas, regardless of whether there are identical nodes there.
	// At the same time need to exclude identical nodes from processing.
	for {
		addrs := traverser.Next()
		if len(addrs) == 0 {
			break
		}

		if n.forEachAddress(ctx, traverser, addrs, f, resErr) {
			break
		}
	}

	if !traverser.Success() {
		var err errIncompletePut
		err.singleErr, _ = resErr.Load().(error)
		return err
	}

	// perform additional container broadcast if needed
	if n.Traversal.submitPrimaryPlacementFinish() {
		err := n.ForEachNode(ctx, f)
		if err != nil {
			n.cfg.Logger.Error(ctx, logs.PutAdditionalContainerBroadcastFailure, zap.Error(err))
			// we don't fail primary operation because of broadcast failure
		}
	}

	return nil
}

func (n *NodeIterator) forEachAddress(ctx context.Context, traverser *placement.Traverser, addrs []placement.Node, f func(context.Context, NodeDescriptor) error, resErr *atomic.Value) bool {
	var wg sync.WaitGroup

	for _, addr := range addrs {
		if ok := n.Exclude[string(addr.PublicKey())]; ok != nil {
			if *ok {
				traverser.SubmitSuccess()
			}
			// This can happen only during additional container broadcast.
			continue
		}

		workerPool, isLocal := n.cfg.getWorkerPool(addr.PublicKey())

		item := new(bool)
		wg.Add(1)
		if err := workerPool.Submit(func() {
			defer wg.Done()

			err := f(ctx, NodeDescriptor{Local: isLocal, Info: addr})
			if err != nil {
				resErr.Store(err)
				svcutil.LogServiceError(n.cfg.Logger, "PUT", addr.Addresses(), err)
				return
			}

			traverser.SubmitSuccess()
			*item = true
		}); err != nil {
			wg.Done()
			svcutil.LogWorkerPoolError(n.cfg.Logger, "PUT", err)
			return true
		}

		// Mark the container node as processed in order to exclude it
		// in subsequent container broadcast. Note that we don't
		// process this node during broadcast if primary placement
		// on it failed.
		n.Traversal.submitProcessed(addr, item)
	}

	wg.Wait()

	return false
}

func NeedAdditionalBroadcast(obj *objectSDK.Object, localOnly bool) bool {
	return len(obj.Children()) > 0 || (!localOnly && (obj.Type() == objectSDK.TypeTombstone || obj.Type() == objectSDK.TypeLock))
}