From a7087faa1ac862f7e3879e70fa80962326311829 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Thu, 3 Dec 2020 15:01:21 +0300 Subject: [PATCH] [#231] shard: Add List method to return all available objects Signed-off-by: Alex Vanin --- pkg/local_object_storage/shard/list.go | 36 ++++++++++++ pkg/local_object_storage/shard/list_test.go | 62 +++++++++++++++++++++ 2 files changed, 98 insertions(+) create mode 100644 pkg/local_object_storage/shard/list.go create mode 100644 pkg/local_object_storage/shard/list_test.go diff --git a/pkg/local_object_storage/shard/list.go b/pkg/local_object_storage/shard/list.go new file mode 100644 index 000000000..7f4f6dbc2 --- /dev/null +++ b/pkg/local_object_storage/shard/list.go @@ -0,0 +1,36 @@ +package shard + +import ( + "fmt" + + "github.com/nspcc-dev/neofs-api-go/pkg/object" + "go.uber.org/zap" +) + +func (s *Shard) List() (*SelectRes, error) { + lst, err := s.metaBase.Containers() + if err != nil { + return nil, fmt.Errorf("can't list stored containers: %w", err) + } + + res := new(SelectRes) + filters := object.NewSearchFilters() + + for i := range lst { + filters = filters[:0] + filters.AddObjectContainerIDFilter(object.MatchStringEqual, lst[i]) + + ids, err := s.metaBase.Select(filters) // consider making List in metabase + if err != nil { + s.log.Debug("can't select all objects", + zap.Stringer("cid", lst[i]), + zap.String("error", err.Error())) + + continue + } + + res.addrList = append(res.addrList, ids...) + } + + return res, nil +} diff --git a/pkg/local_object_storage/shard/list_test.go b/pkg/local_object_storage/shard/list_test.go new file mode 100644 index 000000000..873e8788c --- /dev/null +++ b/pkg/local_object_storage/shard/list_test.go @@ -0,0 +1,62 @@ +package shard_test + +import ( + "testing" + + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard" + "github.com/stretchr/testify/require" +) + +func TestShard_List(t *testing.T) { + sh := newShard(t, false) + shWC := newShard(t, true) + + defer func() { + releaseShard(sh, t) + releaseShard(shWC, t) + }() + + t.Run("without write cache", func(t *testing.T) { + testShardList(t, sh) + }) + + t.Run("with write cache", func(t *testing.T) { + testShardList(t, shWC) + }) +} + +func testShardList(t *testing.T, sh *shard.Shard) { + const C = 10 + const N = 5 + + objs := make(map[string]int) + putPrm := new(shard.PutPrm) + + for i := 0; i < C; i++ { + cid := generateCID() + + for j := 0; j < N; j++ { + obj := generateRawObjectWithCID(t, cid) + addPayload(obj, 1<<2) + + objs[obj.Object().Address().String()] = 0 + + putPrm.WithObject(obj.Object()) + + _, err := sh.Put(putPrm) + require.NoError(t, err) + } + } + + res, err := sh.List() + require.NoError(t, err) + + for _, objID := range res.AddressList() { + + i, ok := objs[objID.String()] + require.True(t, ok) + require.Equal(t, 0, i) + + objs[objID.String()] = 1 + } +}