frostfs-node/pkg/local_object_storage/shard/list.go

package shard

import (
	"context"
	"fmt"

	"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
	objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
	meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase"
	tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing"
	"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
	cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
	objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
	"go.opentelemetry.io/otel/attribute"
	"go.opentelemetry.io/otel/trace"
	"go.uber.org/zap"
)

// Cursor is a type for continuous object listing.
type Cursor = meta.Cursor

// ErrEndOfListing is returned from object listing with cursor
// when storage can't return any more objects after provided
// cursor. Use nil cursor object to start listing again.
var ErrEndOfListing = meta.ErrEndOfListing

type ListContainersPrm struct{}

type ListContainersRes struct {
	containers []cid.ID
}

func (r ListContainersRes) Containers() []cid.ID {
	return r.containers
}

// IterateOverContainersPrm contains parameters for IterateOverContainers operation.
type IterateOverContainersPrm struct {
	// Handler function executed upon containers in db.
	Handler func(context.Context, []byte, cid.ID) error
}

// IterateOverObjectsInContainerPrm contains parameters for IterateOverObjectsInContainer operation.
type IterateOverObjectsInContainerPrm struct {
	// BucketName container's bucket name.
	BucketName []byte
	// Handler function executed upon objects in db.
	Handler func(context.Context, *objectcore.Info) error
}

// CountAliveObjectsInBucketPrm contains parameters for CountAliveObjectsInBucket operation.
type CountAliveObjectsInBucketPrm struct {
	// BucketName container's bucket name.
	BucketName []byte
}

// ListWithCursorPrm contains parameters for ListWithCursor operation.
type ListWithCursorPrm struct {
	count  uint32
	cursor *Cursor
}

// ListWithCursorRes contains values returned from ListWithCursor operation.
type ListWithCursorRes struct {
	addrList []objectcore.Info
	cursor   *Cursor
}

// WithCount sets maximum amount of addresses that ListWithCursor should return.
func (p *ListWithCursorPrm) WithCount(count uint32) {
	p.count = count
}

// WithCursor sets cursor for ListWithCursor operation. For initial request,
// ignore this param or use nil value. For consecutive requests, use value
// from ListWithCursorRes.
func (p *ListWithCursorPrm) WithCursor(cursor *Cursor) {
	p.cursor = cursor
}

// AddressList returns addresses selected by ListWithCursor operation.
func (r ListWithCursorRes) AddressList() []objectcore.Info {
	return r.addrList
}

// Cursor returns cursor for consecutive listing requests.
func (r ListWithCursorRes) Cursor() *Cursor {
	return r.cursor
}

// List returns all objects physically stored in the Shard.
func (s *Shard) List(ctx context.Context) (res SelectRes, err error) {
	ctx, span := tracing.StartSpanFromContext(ctx, "Shard.List",
		trace.WithAttributes(
			attribute.String("shard_id", s.ID().String()),
		))
	defer span.End()

	s.m.RLock()
	defer s.m.RUnlock()

	if s.info.Mode.NoMetabase() {
		return SelectRes{}, ErrDegradedMode
	}

	lst, err := s.metaBase.Containers(ctx)
	if err != nil {
		return res, fmt.Errorf("can't list stored containers: %w", err)
	}

	filters := objectSDK.NewSearchFilters()
	filters.AddPhyFilter()

	for i := range lst {
		var sPrm meta.SelectPrm
		sPrm.SetContainerID(lst[i])
		sPrm.SetFilters(filters)

		sRes, err := s.metaBase.Select(ctx, sPrm) // consider making List in metabase
		if err != nil {
			s.log.Debug(logs.ShardCantSelectAllObjects,
				zap.Stringer("cid", lst[i]),
				zap.String("error", err.Error()),
				zap.String("trace_id", tracingPkg.GetTraceID(ctx)))

			continue
		}

		res.addrList = append(res.addrList, sRes.AddressList()...)
	}

	return res, nil
}

func (s *Shard) ListContainers(ctx context.Context, _ ListContainersPrm) (ListContainersRes, error) {
	ctx, span := tracing.StartSpanFromContext(ctx, "Shard.ListContainers",
		trace.WithAttributes(
			attribute.String("shard_id", s.ID().String()),
		))
	defer span.End()

	if s.GetMode().NoMetabase() {
		return ListContainersRes{}, ErrDegradedMode
	}

	containers, err := s.metaBase.Containers(ctx)
	if err != nil {
		return ListContainersRes{}, fmt.Errorf("could not get list of containers: %w", err)
	}

	return ListContainersRes{
		containers: containers,
	}, nil
}

// ListWithCursor lists physical objects available in shard starting from
// cursor. Includes regular, tombstone and storage group objects. Does not
// include inhumed objects. Use cursor value from response for consecutive requests.
//
// Returns ErrEndOfListing if there are no more objects to return or count
// parameter set to zero.
func (s *Shard) ListWithCursor(ctx context.Context, prm ListWithCursorPrm) (ListWithCursorRes, error) {
	_, span := tracing.StartSpanFromContext(ctx, "shard.ListWithCursor",
		trace.WithAttributes(
			attribute.Int64("count", int64(prm.count)),
			attribute.Bool("has_cursor", prm.cursor != nil),
		))
	defer span.End()

	if s.GetMode().NoMetabase() {
		return ListWithCursorRes{}, ErrDegradedMode
	}

	var metaPrm meta.ListPrm
	metaPrm.SetCount(prm.count)
	metaPrm.SetCursor(prm.cursor)
	res, err := s.metaBase.ListWithCursor(ctx, metaPrm)
	if err != nil {
		return ListWithCursorRes{}, fmt.Errorf("could not get list of objects: %w", err)
	}

	return ListWithCursorRes{
		addrList: res.AddressList(),
		cursor:   res.Cursor(),
	}, nil
}

// IterateOverContainers lists physical containers presented in shard.
func (s *Shard) IterateOverContainers(ctx context.Context, prm IterateOverContainersPrm) error {
	_, span := tracing.StartSpanFromContext(ctx, "shard.IterateOverContainers",
		trace.WithAttributes(
			attribute.Bool("has_handler", prm.Handler != nil),
		))
	defer span.End()

	s.m.RLock()
	defer s.m.RUnlock()

	if s.info.Mode.NoMetabase() {
		return ErrDegradedMode
	}

	var metaPrm meta.IterateOverContainersPrm
	metaPrm.Handler = prm.Handler
	err := s.metaBase.IterateOverContainers(ctx, metaPrm)
	if err != nil {
		return fmt.Errorf("could not iterate over containers: %w", err)
	}

	return nil
}

// IterateOverObjectsInContainer lists physical objects presented in shard for provided container's bucket name.
func (s *Shard) IterateOverObjectsInContainer(ctx context.Context, prm IterateOverObjectsInContainerPrm) error {
	_, span := tracing.StartSpanFromContext(ctx, "shard.IterateOverObjectsInContainer",
		trace.WithAttributes(
			attribute.Bool("has_handler", prm.Handler != nil),
		))
	defer span.End()

	s.m.RLock()
	defer s.m.RUnlock()

	if s.info.Mode.NoMetabase() {
		return ErrDegradedMode
	}

	var metaPrm meta.IterateOverObjectsInContainerPrm
	metaPrm.BucketName = prm.BucketName
	metaPrm.Handler = prm.Handler
	err := s.metaBase.IterateOverObjectsInContainer(ctx, metaPrm)
	if err != nil {
		return fmt.Errorf("could not iterate over objects: %w", err)
	}

	return nil
}

// CountAliveObjectsInBucket count objects in bucket which aren't in graveyard or garbage.
func (s *Shard) CountAliveObjectsInBucket(ctx context.Context, prm CountAliveObjectsInBucketPrm) (uint64, error) {
	_, span := tracing.StartSpanFromContext(ctx, "shard.CountAliveObjectsInBucket")
	defer span.End()

	s.m.RLock()
	defer s.m.RUnlock()

	if s.info.Mode.NoMetabase() {
		return 0, ErrDegradedMode
	}

	var metaPrm meta.CountAliveObjectsInBucketPrm
	metaPrm.BucketName = prm.BucketName
	count, err := s.metaBase.CountAliveObjectsInBucket(ctx, metaPrm)
	if err != nil {
		return 0, fmt.Errorf("could not count alive objects in bucket: %w", err)
	}

	return count, nil
}