forked from TrueCloudLab/frostfs-node
[#789] blobovnicza: Implement iterator
There is a need to be able to process all stored objects saved in `Blobovnicza`. Implement `Blobovnicza.Iterate` method which iterates over all objects. Implement `IterateObjects` helper function to simplify the code. Signed-off-by: Leonard Lyubich <leonard@nspcc.ru>
This commit is contained in:
parent
55c94a0152
commit
b618a44d69
2 changed files with 121 additions and 0 deletions
|
@ -2,6 +2,7 @@ package blobovnicza
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
|
"encoding/binary"
|
||||||
"errors"
|
"errors"
|
||||||
"math/rand"
|
"math/rand"
|
||||||
"os"
|
"os"
|
||||||
|
@ -9,6 +10,7 @@ import (
|
||||||
|
|
||||||
cidtest "github.com/nspcc-dev/neofs-api-go/pkg/container/id/test"
|
cidtest "github.com/nspcc-dev/neofs-api-go/pkg/container/id/test"
|
||||||
objectSDK "github.com/nspcc-dev/neofs-api-go/pkg/object"
|
objectSDK "github.com/nspcc-dev/neofs-api-go/pkg/object"
|
||||||
|
objecttest "github.com/nspcc-dev/neofs-api-go/pkg/object/test"
|
||||||
"github.com/nspcc-dev/neofs-node/pkg/core/object"
|
"github.com/nspcc-dev/neofs-node/pkg/core/object"
|
||||||
"github.com/nspcc-dev/neofs-node/pkg/util/logger/test"
|
"github.com/nspcc-dev/neofs-node/pkg/util/logger/test"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
|
@ -118,3 +120,57 @@ func TestBlobovnicza(t *testing.T) {
|
||||||
|
|
||||||
require.NoError(t, blz.Close())
|
require.NoError(t, blz.Close())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestIterateObjects(t *testing.T) {
|
||||||
|
p := t.Name()
|
||||||
|
|
||||||
|
// create Blobovnicza instance
|
||||||
|
blz := New(
|
||||||
|
WithPath(p),
|
||||||
|
WithObjectSizeLimit(1<<10),
|
||||||
|
WithFullSizeLimit(100<<10),
|
||||||
|
)
|
||||||
|
|
||||||
|
defer os.Remove(p)
|
||||||
|
|
||||||
|
// open Blobovnicza
|
||||||
|
require.NoError(t, blz.Open())
|
||||||
|
|
||||||
|
// initialize Blobovnicza
|
||||||
|
require.NoError(t, blz.Init())
|
||||||
|
|
||||||
|
const objNum = 5
|
||||||
|
|
||||||
|
mObjs := make(map[string][]byte)
|
||||||
|
|
||||||
|
for i := uint64(0); i < objNum; i++ {
|
||||||
|
data := make([]byte, 8) // actual data doesn't really matter for test
|
||||||
|
|
||||||
|
binary.BigEndian.PutUint64(data, i)
|
||||||
|
|
||||||
|
mObjs[string(data)] = data
|
||||||
|
}
|
||||||
|
|
||||||
|
var putPrm PutPrm
|
||||||
|
|
||||||
|
for _, v := range mObjs {
|
||||||
|
putPrm.SetAddress(objecttest.Address())
|
||||||
|
putPrm.SetMarshaledObject(v)
|
||||||
|
|
||||||
|
_, err := blz.Put(&putPrm)
|
||||||
|
require.NoError(t, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err := IterateObjects(blz, func(data []byte) error {
|
||||||
|
v, ok := mObjs[string(data)]
|
||||||
|
require.True(t, ok)
|
||||||
|
|
||||||
|
require.Equal(t, v, data)
|
||||||
|
|
||||||
|
delete(mObjs, string(data))
|
||||||
|
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Empty(t, mObjs)
|
||||||
|
}
|
||||||
|
|
|
@ -55,3 +55,68 @@ func max(a, b uint64) uint64 {
|
||||||
|
|
||||||
return b
|
return b
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// IterationElement represents a unit of elements through which Iterate operation passes.
|
||||||
|
type IterationElement struct {
|
||||||
|
data []byte
|
||||||
|
}
|
||||||
|
|
||||||
|
// ObjectData returns stored object in a binary representation.
|
||||||
|
func (x IterationElement) ObjectData() []byte {
|
||||||
|
return x.data
|
||||||
|
}
|
||||||
|
|
||||||
|
// IterationHandler is a generic processor of IterationElement.
|
||||||
|
type IterationHandler func(IterationElement) error
|
||||||
|
|
||||||
|
// IteratePrm groups the parameters of Iterate operation.
|
||||||
|
type IteratePrm struct {
|
||||||
|
handler IterationHandler
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetHandler sets handler to be called iteratively.
|
||||||
|
func (x *IteratePrm) SetHandler(h IterationHandler) {
|
||||||
|
x.handler = h
|
||||||
|
}
|
||||||
|
|
||||||
|
// IterateRes groups resulting values of Iterate operation.
|
||||||
|
type IterateRes struct {
|
||||||
|
}
|
||||||
|
|
||||||
|
// Iterate goes through all stored objects, and passes their headers
|
||||||
|
// to parameterized handler until error return.
|
||||||
|
//
|
||||||
|
// Returns handler's errors directly. Returns nil after iterating finish.
|
||||||
|
//
|
||||||
|
// Handler should not retain object data. Handler must not be nil.
|
||||||
|
func (b *Blobovnicza) Iterate(prm IteratePrm) (*IterateRes, error) {
|
||||||
|
var elem IterationElement
|
||||||
|
|
||||||
|
if err := b.boltDB.View(func(tx *bbolt.Tx) error {
|
||||||
|
return b.iterateBuckets(tx, func(lower, upper uint64, buck *bbolt.Bucket) (bool, error) {
|
||||||
|
err := buck.ForEach(func(k, v []byte) error {
|
||||||
|
elem.data = v
|
||||||
|
return prm.handler(elem)
|
||||||
|
})
|
||||||
|
|
||||||
|
return err != nil, err
|
||||||
|
})
|
||||||
|
}); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return new(IterateRes), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// IterateObjects is a helper function which iterates over Blobovnicza and passes binary objects to f.
|
||||||
|
func IterateObjects(blz *Blobovnicza, f func([]byte) error) error {
|
||||||
|
var prm IteratePrm
|
||||||
|
|
||||||
|
prm.SetHandler(func(elem IterationElement) error {
|
||||||
|
return f(elem.ObjectData())
|
||||||
|
})
|
||||||
|
|
||||||
|
_, err := blz.Iterate(prm)
|
||||||
|
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
Loading…
Reference in a new issue