[#789] blobovnicza: Implement iterator

There is a need to be able to process all stored objects saved in
`Blobovnicza`.

Implement `Blobovnicza.Iterate` method which iterates over all objects.
Implement `IterateObjects` helper function to simplify the code.

Signed-off-by: Leonard Lyubich <leonard@nspcc.ru>
This commit is contained in:
Leonard Lyubich 2021-09-13 13:13:18 +03:00 committed by Alex Vanin
parent 55c94a0152
commit b618a44d69
2 changed files with 121 additions and 0 deletions

View file

@ -2,6 +2,7 @@ package blobovnicza
import ( import (
"crypto/sha256" "crypto/sha256"
"encoding/binary"
"errors" "errors"
"math/rand" "math/rand"
"os" "os"
@ -9,6 +10,7 @@ import (
cidtest "github.com/nspcc-dev/neofs-api-go/pkg/container/id/test" cidtest "github.com/nspcc-dev/neofs-api-go/pkg/container/id/test"
objectSDK "github.com/nspcc-dev/neofs-api-go/pkg/object" objectSDK "github.com/nspcc-dev/neofs-api-go/pkg/object"
objecttest "github.com/nspcc-dev/neofs-api-go/pkg/object/test"
"github.com/nspcc-dev/neofs-node/pkg/core/object" "github.com/nspcc-dev/neofs-node/pkg/core/object"
"github.com/nspcc-dev/neofs-node/pkg/util/logger/test" "github.com/nspcc-dev/neofs-node/pkg/util/logger/test"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
@ -118,3 +120,57 @@ func TestBlobovnicza(t *testing.T) {
require.NoError(t, blz.Close()) require.NoError(t, blz.Close())
} }
func TestIterateObjects(t *testing.T) {
p := t.Name()
// create Blobovnicza instance
blz := New(
WithPath(p),
WithObjectSizeLimit(1<<10),
WithFullSizeLimit(100<<10),
)
defer os.Remove(p)
// open Blobovnicza
require.NoError(t, blz.Open())
// initialize Blobovnicza
require.NoError(t, blz.Init())
const objNum = 5
mObjs := make(map[string][]byte)
for i := uint64(0); i < objNum; i++ {
data := make([]byte, 8) // actual data doesn't really matter for test
binary.BigEndian.PutUint64(data, i)
mObjs[string(data)] = data
}
var putPrm PutPrm
for _, v := range mObjs {
putPrm.SetAddress(objecttest.Address())
putPrm.SetMarshaledObject(v)
_, err := blz.Put(&putPrm)
require.NoError(t, err)
}
err := IterateObjects(blz, func(data []byte) error {
v, ok := mObjs[string(data)]
require.True(t, ok)
require.Equal(t, v, data)
delete(mObjs, string(data))
return nil
})
require.NoError(t, err)
require.Empty(t, mObjs)
}

View file

@ -55,3 +55,68 @@ func max(a, b uint64) uint64 {
return b return b
} }
// IterationElement represents a unit of elements through which Iterate operation passes.
type IterationElement struct {
data []byte
}
// ObjectData returns stored object in a binary representation.
func (x IterationElement) ObjectData() []byte {
return x.data
}
// IterationHandler is a generic processor of IterationElement.
type IterationHandler func(IterationElement) error
// IteratePrm groups the parameters of Iterate operation.
type IteratePrm struct {
handler IterationHandler
}
// SetHandler sets handler to be called iteratively.
func (x *IteratePrm) SetHandler(h IterationHandler) {
x.handler = h
}
// IterateRes groups resulting values of Iterate operation.
type IterateRes struct {
}
// Iterate goes through all stored objects, and passes their headers
// to parameterized handler until error return.
//
// Returns handler's errors directly. Returns nil after iterating finish.
//
// Handler should not retain object data. Handler must not be nil.
func (b *Blobovnicza) Iterate(prm IteratePrm) (*IterateRes, error) {
var elem IterationElement
if err := b.boltDB.View(func(tx *bbolt.Tx) error {
return b.iterateBuckets(tx, func(lower, upper uint64, buck *bbolt.Bucket) (bool, error) {
err := buck.ForEach(func(k, v []byte) error {
elem.data = v
return prm.handler(elem)
})
return err != nil, err
})
}); err != nil {
return nil, err
}
return new(IterateRes), nil
}
// IterateObjects is a helper function which iterates over Blobovnicza and passes binary objects to f.
func IterateObjects(blz *Blobovnicza, f func([]byte) error) error {
var prm IteratePrm
prm.SetHandler(func(elem IterationElement) error {
return f(elem.ObjectData())
})
_, err := blz.Iterate(prm)
return err
}