forked from TrueCloudLab/frostfs-node
[#789] blobovnicza: Implement iterator
There is a need to be able to process all stored objects saved in `Blobovnicza`. Implement `Blobovnicza.Iterate` method which iterates over all objects. Implement `IterateObjects` helper function to simplify the code. Signed-off-by: Leonard Lyubich <leonard@nspcc.ru>
This commit is contained in:
parent
55c94a0152
commit
b618a44d69
2 changed files with 121 additions and 0 deletions
|
@ -2,6 +2,7 @@ package blobovnicza
|
|||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"math/rand"
|
||||
"os"
|
||||
|
@ -9,6 +10,7 @@ import (
|
|||
|
||||
cidtest "github.com/nspcc-dev/neofs-api-go/pkg/container/id/test"
|
||||
objectSDK "github.com/nspcc-dev/neofs-api-go/pkg/object"
|
||||
objecttest "github.com/nspcc-dev/neofs-api-go/pkg/object/test"
|
||||
"github.com/nspcc-dev/neofs-node/pkg/core/object"
|
||||
"github.com/nspcc-dev/neofs-node/pkg/util/logger/test"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
@ -118,3 +120,57 @@ func TestBlobovnicza(t *testing.T) {
|
|||
|
||||
require.NoError(t, blz.Close())
|
||||
}
|
||||
|
||||
func TestIterateObjects(t *testing.T) {
|
||||
p := t.Name()
|
||||
|
||||
// create Blobovnicza instance
|
||||
blz := New(
|
||||
WithPath(p),
|
||||
WithObjectSizeLimit(1<<10),
|
||||
WithFullSizeLimit(100<<10),
|
||||
)
|
||||
|
||||
defer os.Remove(p)
|
||||
|
||||
// open Blobovnicza
|
||||
require.NoError(t, blz.Open())
|
||||
|
||||
// initialize Blobovnicza
|
||||
require.NoError(t, blz.Init())
|
||||
|
||||
const objNum = 5
|
||||
|
||||
mObjs := make(map[string][]byte)
|
||||
|
||||
for i := uint64(0); i < objNum; i++ {
|
||||
data := make([]byte, 8) // actual data doesn't really matter for test
|
||||
|
||||
binary.BigEndian.PutUint64(data, i)
|
||||
|
||||
mObjs[string(data)] = data
|
||||
}
|
||||
|
||||
var putPrm PutPrm
|
||||
|
||||
for _, v := range mObjs {
|
||||
putPrm.SetAddress(objecttest.Address())
|
||||
putPrm.SetMarshaledObject(v)
|
||||
|
||||
_, err := blz.Put(&putPrm)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
err := IterateObjects(blz, func(data []byte) error {
|
||||
v, ok := mObjs[string(data)]
|
||||
require.True(t, ok)
|
||||
|
||||
require.Equal(t, v, data)
|
||||
|
||||
delete(mObjs, string(data))
|
||||
|
||||
return nil
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, mObjs)
|
||||
}
|
||||
|
|
|
@ -55,3 +55,68 @@ func max(a, b uint64) uint64 {
|
|||
|
||||
return b
|
||||
}
|
||||
|
||||
// IterationElement represents a unit of elements through which Iterate operation passes.
|
||||
type IterationElement struct {
|
||||
data []byte
|
||||
}
|
||||
|
||||
// ObjectData returns stored object in a binary representation.
|
||||
func (x IterationElement) ObjectData() []byte {
|
||||
return x.data
|
||||
}
|
||||
|
||||
// IterationHandler is a generic processor of IterationElement.
|
||||
type IterationHandler func(IterationElement) error
|
||||
|
||||
// IteratePrm groups the parameters of Iterate operation.
|
||||
type IteratePrm struct {
|
||||
handler IterationHandler
|
||||
}
|
||||
|
||||
// SetHandler sets handler to be called iteratively.
|
||||
func (x *IteratePrm) SetHandler(h IterationHandler) {
|
||||
x.handler = h
|
||||
}
|
||||
|
||||
// IterateRes groups resulting values of Iterate operation.
|
||||
type IterateRes struct {
|
||||
}
|
||||
|
||||
// Iterate goes through all stored objects, and passes their headers
|
||||
// to parameterized handler until error return.
|
||||
//
|
||||
// Returns handler's errors directly. Returns nil after iterating finish.
|
||||
//
|
||||
// Handler should not retain object data. Handler must not be nil.
|
||||
func (b *Blobovnicza) Iterate(prm IteratePrm) (*IterateRes, error) {
|
||||
var elem IterationElement
|
||||
|
||||
if err := b.boltDB.View(func(tx *bbolt.Tx) error {
|
||||
return b.iterateBuckets(tx, func(lower, upper uint64, buck *bbolt.Bucket) (bool, error) {
|
||||
err := buck.ForEach(func(k, v []byte) error {
|
||||
elem.data = v
|
||||
return prm.handler(elem)
|
||||
})
|
||||
|
||||
return err != nil, err
|
||||
})
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return new(IterateRes), nil
|
||||
}
|
||||
|
||||
// IterateObjects is a helper function which iterates over Blobovnicza and passes binary objects to f.
|
||||
func IterateObjects(blz *Blobovnicza, f func([]byte) error) error {
|
||||
var prm IteratePrm
|
||||
|
||||
prm.SetHandler(func(elem IterationElement) error {
|
||||
return f(elem.ObjectData())
|
||||
})
|
||||
|
||||
_, err := blz.Iterate(prm)
|
||||
|
||||
return err
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue