From de1f601765d63fddcf8d9d3a2ff5fd94c16671ac Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Fri, 22 Jan 2021 16:21:45 +0300 Subject: [PATCH] [#337] engine: Add container listing for both engine and shard Container listing already supported in the metabase for `engine.List` operation. To get container statistics engine should provide both the option to get container volume estimation and list of all containers. Signed-off-by: Alex Vanin --- pkg/local_object_storage/engine/container.go | 53 ++++++++++++++++++++ pkg/local_object_storage/shard/list.go | 32 ++++++++++++ 2 files changed, 85 insertions(+) diff --git a/pkg/local_object_storage/engine/container.go b/pkg/local_object_storage/engine/container.go index 008cea4eb..11766bac4 100644 --- a/pkg/local_object_storage/engine/container.go +++ b/pkg/local_object_storage/engine/container.go @@ -14,6 +14,12 @@ type ContainerSizeRes struct { size uint64 } +type ListContainersPrm struct{} + +type ListContainersRes struct { + containers []*container.ID +} + func (p *ContainerSizePrm) WithContainerID(cid *container.ID) *ContainerSizePrm { if p != nil { p.cid = cid @@ -26,6 +32,10 @@ func (r *ContainerSizeRes) Size() uint64 { return r.size } +func (r *ListContainersRes) Containers() []*container.ID { + return r.containers +} + // ContainerSize returns sum of estimation container sizes among all shards. func (e *StorageEngine) ContainerSize(prm *ContainerSizePrm) *ContainerSizeRes { return &ContainerSizeRes{ @@ -57,3 +67,46 @@ func (e *StorageEngine) containerSize(id *container.ID) (total uint64) { return total } + +// ListContainers returns unique container IDs presented in the engine objects. +func (e *StorageEngine) ListContainers(_ *ListContainersPrm) *ListContainersRes { + return &ListContainersRes{ + containers: e.listContainers(), + } +} + +// ListContainers returns unique container IDs presented in the engine objects. +func ListContainers(e *StorageEngine) []*container.ID { + return e.ListContainers(&ListContainersPrm{}).Containers() +} + +func (e *StorageEngine) listContainers() []*container.ID { + uniqueIDs := make(map[string]*container.ID) + + e.iterateOverUnsortedShards(func(s *shard.Shard) (stop bool) { + cnrs, err := shard.ListContainers(s) + if err != nil { + e.log.Warn("can't get list of containers", + zap.Stringer("shard_id", s.ID()), + zap.String("error", err.Error())) + + return false + } + + for i := range cnrs { + id := cnrs[i].String() + if _, ok := uniqueIDs[id]; !ok { + uniqueIDs[id] = cnrs[i] + } + } + + return false + }) + + result := make([]*container.ID, 0, len(uniqueIDs)) + for _, v := range uniqueIDs { + result = append(result, v) + } + + return result +} diff --git a/pkg/local_object_storage/shard/list.go b/pkg/local_object_storage/shard/list.go index 24dc5c18a..9dfeb6276 100644 --- a/pkg/local_object_storage/shard/list.go +++ b/pkg/local_object_storage/shard/list.go @@ -3,11 +3,23 @@ package shard import ( "fmt" + "github.com/nspcc-dev/neofs-api-go/pkg/container" "github.com/nspcc-dev/neofs-api-go/pkg/object" meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase" + "github.com/pkg/errors" "go.uber.org/zap" ) +type ListContainersPrm struct{} + +type ListContainersRes struct { + containers []*container.ID +} + +func (r *ListContainersRes) Containers() []*container.ID { + return r.containers +} + func (s *Shard) List() (*SelectRes, error) { lst, err := s.metaBase.Containers() if err != nil { @@ -32,3 +44,23 @@ func (s *Shard) List() (*SelectRes, error) { return res, nil } + +func (s *Shard) ListContainers(_ *ListContainersPrm) (*ListContainersRes, error) { + containers, err := s.metaBase.Containers() + if err != nil { + return nil, errors.Wrap(err, "could not get list of containers") + } + + return &ListContainersRes{ + containers: containers, + }, nil +} + +func ListContainers(s *Shard) ([]*container.ID, error) { + res, err := s.ListContainers(&ListContainersPrm{}) + if err != nil { + return nil, err + } + + return res.Containers(), nil +}