From b618a44d69a75d01ec6ec642c3e1cef809b01f13 Mon Sep 17 00:00:00 2001 From: Leonard Lyubich Date: Mon, 13 Sep 2021 13:13:18 +0300 Subject: [PATCH] [#789] blobovnicza: Implement iterator There is a need to be able to process all stored objects saved in `Blobovnicza`. Implement `Blobovnicza.Iterate` method which iterates over all objects. Implement `IterateObjects` helper function to simplify the code. Signed-off-by: Leonard Lyubich --- .../blobovnicza/blobovnicza_test.go | 56 ++++++++++++++++ .../blobovnicza/iterate.go | 65 +++++++++++++++++++ 2 files changed, 121 insertions(+) diff --git a/pkg/local_object_storage/blobovnicza/blobovnicza_test.go b/pkg/local_object_storage/blobovnicza/blobovnicza_test.go index e4add96ae..312a01a44 100644 --- a/pkg/local_object_storage/blobovnicza/blobovnicza_test.go +++ b/pkg/local_object_storage/blobovnicza/blobovnicza_test.go @@ -2,6 +2,7 @@ package blobovnicza import ( "crypto/sha256" + "encoding/binary" "errors" "math/rand" "os" @@ -9,6 +10,7 @@ import ( cidtest "github.com/nspcc-dev/neofs-api-go/pkg/container/id/test" objectSDK "github.com/nspcc-dev/neofs-api-go/pkg/object" + objecttest "github.com/nspcc-dev/neofs-api-go/pkg/object/test" "github.com/nspcc-dev/neofs-node/pkg/core/object" "github.com/nspcc-dev/neofs-node/pkg/util/logger/test" "github.com/stretchr/testify/require" @@ -118,3 +120,57 @@ func TestBlobovnicza(t *testing.T) { require.NoError(t, blz.Close()) } + +func TestIterateObjects(t *testing.T) { + p := t.Name() + + // create Blobovnicza instance + blz := New( + WithPath(p), + WithObjectSizeLimit(1<<10), + WithFullSizeLimit(100<<10), + ) + + defer os.Remove(p) + + // open Blobovnicza + require.NoError(t, blz.Open()) + + // initialize Blobovnicza + require.NoError(t, blz.Init()) + + const objNum = 5 + + mObjs := make(map[string][]byte) + + for i := uint64(0); i < objNum; i++ { + data := make([]byte, 8) // actual data doesn't really matter for test + + binary.BigEndian.PutUint64(data, i) + + mObjs[string(data)] = data + } + + var putPrm PutPrm + + for _, v := range mObjs { + putPrm.SetAddress(objecttest.Address()) + putPrm.SetMarshaledObject(v) + + _, err := blz.Put(&putPrm) + require.NoError(t, err) + } + + err := IterateObjects(blz, func(data []byte) error { + v, ok := mObjs[string(data)] + require.True(t, ok) + + require.Equal(t, v, data) + + delete(mObjs, string(data)) + + return nil + }) + require.NoError(t, err) + require.Empty(t, mObjs) +} diff --git a/pkg/local_object_storage/blobovnicza/iterate.go b/pkg/local_object_storage/blobovnicza/iterate.go index 715fcd929..fc9379e82 100644 --- a/pkg/local_object_storage/blobovnicza/iterate.go +++ b/pkg/local_object_storage/blobovnicza/iterate.go @@ -55,3 +55,68 @@ func max(a, b uint64) uint64 { return b } + +// IterationElement represents a unit of elements through which Iterate operation passes. +type IterationElement struct { + data []byte +} + +// ObjectData returns stored object in a binary representation. +func (x IterationElement) ObjectData() []byte { + return x.data +} + +// IterationHandler is a generic processor of IterationElement. +type IterationHandler func(IterationElement) error + +// IteratePrm groups the parameters of Iterate operation. +type IteratePrm struct { + handler IterationHandler +} + +// SetHandler sets handler to be called iteratively. +func (x *IteratePrm) SetHandler(h IterationHandler) { + x.handler = h +} + +// IterateRes groups resulting values of Iterate operation. +type IterateRes struct { +} + +// Iterate goes through all stored objects, and passes their headers +// to parameterized handler until error return. +// +// Returns handler's errors directly. Returns nil after iterating finish. +// +// Handler should not retain object data. Handler must not be nil. +func (b *Blobovnicza) Iterate(prm IteratePrm) (*IterateRes, error) { + var elem IterationElement + + if err := b.boltDB.View(func(tx *bbolt.Tx) error { + return b.iterateBuckets(tx, func(lower, upper uint64, buck *bbolt.Bucket) (bool, error) { + err := buck.ForEach(func(k, v []byte) error { + elem.data = v + return prm.handler(elem) + }) + + return err != nil, err + }) + }); err != nil { + return nil, err + } + + return new(IterateRes), nil +} + +// IterateObjects is a helper function which iterates over Blobovnicza and passes binary objects to f. +func IterateObjects(blz *Blobovnicza, f func([]byte) error) error { + var prm IteratePrm + + prm.SetHandler(func(elem IterationElement) error { + return f(elem.ObjectData()) + }) + + _, err := blz.Iterate(prm) + + return err +}