realloc/frostfs-node
1
0
Fork 0
forked from TrueCloudLab/frostfs-node
Code Releases Packages Activity

[#1520] shard: Ignore errors on metabase refill

Browse source 
Signed-off-by: Evgenii Stratonikov <evgeniy@nspcc.ru>
This commit is contained in:
Evgenii Stratonikov 2022-06-21 10:52:03 +03:00 • committed by fyrchik
parent 78ea450c25
commit 7df50297cd
7 changed files with 108 additions and 27 deletions
Download patch file Download diff file Expand all files Collapse all files

9
pkg/local_object_storage/blobstor/fstree/fstree.go
View file

@ -72,6 +72,7 @@ func addressFromString(s string) (*oid.Address, error) {
type IterationPrm struct {
handler func(addr oid.Address, data []byte) error
ignoreErrors bool
errorHandler func(oid.Address, error) error
lazyHandler func(oid.Address, func() ([]byte, error)) error
}
@ -92,6 +93,11 @@ func (p *IterationPrm) WithIgnoreErrors(ignore bool) {
p.ignoreErrors = ignore
}
// WithErrorHandler sets error handler for objects that cannot be read or unmarshaled.
func (p *IterationPrm) WithErrorHandler(f func(oid.Address, error) error) {
p.errorHandler = f
}
// Iterate iterates over all stored objects.
func (t *FSTree) Iterate(prm IterationPrm) error {
return t.iterate(0, []string{t.RootPath}, prm)
@ -141,6 +147,9 @@ func (t *FSTree) iterate(depth int, curPath []string, prm IterationPrm) error {
data, err = os.ReadFile(filepath.Join(curPath...))
if err != nil {
if prm.ignoreErrors {
if prm.errorHandler != nil {
return prm.errorHandler(*addr, err)
}
continue
}
return err

57
pkg/local_object_storage/blobstor/iterate.go
View file

@ -5,14 +5,16 @@ import (
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree"
"github.com/nspcc-dev/neofs-sdk-go/object"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"go.uber.org/zap"
)
// IterationElement represents a unit of elements through which Iterate operation passes.
type IterationElement struct {
data []byte
addr oid.Address
blzID *blobovnicza.ID
}
@ -27,6 +29,11 @@ func (x IterationElement) BlobovniczaID() *blobovnicza.ID {
return x.blzID
}
// Address returns the object address.
func (x IterationElement) Address() oid.Address {
return x.addr
}
// IterationHandler is a generic processor of IterationElement.
type IterationHandler func(IterationElement) error
@ -34,6 +41,7 @@ type IterationHandler func(IterationElement) error
type IteratePrm struct {
handler IterationHandler
ignoreErrors bool
errorHandler func(oid.Address, error) error
}
// IterateRes groups the resulting values of Iterate operation.
@ -49,6 +57,11 @@ func (i *IteratePrm) IgnoreErrors() {
i.ignoreErrors = true
}
// SetErrorHandler sets error handler for objects that cannot be read or unmarshaled.
func (i *IteratePrm) SetErrorHandler(f func(oid.Address, error) error) {
i.errorHandler = f
}
// Iterate traverses the storage over the stored objects and calls the handler
// on each element.
//
@ -60,18 +73,22 @@ func (b *BlobStor) Iterate(prm IteratePrm) (IterateRes, error) {
var elem IterationElement
err := b.blobovniczas.iterateBlobovniczas(prm.ignoreErrors, func(p string, blz *blobovnicza.Blobovnicza) error {
err := blobovnicza.IterateObjects(blz, func(data []byte) error {
err := blobovnicza.IterateObjects(blz, func(addr oid.Address, data []byte) error {
var err error
// decompress the data
elem.data, err = b.decompressor(data)
if err != nil {
if prm.ignoreErrors {
if prm.errorHandler != nil {
return prm.errorHandler(addr, err)
}
return nil
}
return fmt.Errorf("could not decompress object data: %w", err)
}
elem.addr = addr
elem.blzID = blobovnicza.NewIDFromBytes([]byte(p))
return prm.handler(elem)
@ -90,16 +107,21 @@ func (b *BlobStor) Iterate(prm IteratePrm) (IterateRes, error) {
var fsPrm fstree.IterationPrm
fsPrm.WithIgnoreErrors(prm.ignoreErrors)
fsPrm.WithHandler(func(_ oid.Address, data []byte) error {
fsPrm.WithHandler(func(addr oid.Address, data []byte) error {
// decompress the data
elem.data, err = b.decompressor(data)
if err != nil {
if prm.ignoreErrors {
if prm.errorHandler != nil {
return prm.errorHandler(addr, err)
}
return nil
}
return fmt.Errorf("could not decompress object data: %w", err)
}
elem.addr = addr
return prm.handler(elem)
})
@ -113,31 +135,22 @@ func (b *BlobStor) Iterate(prm IteratePrm) (IterateRes, error) {
}
// IterateBinaryObjects is a helper function which iterates over BlobStor and passes binary objects to f.
func IterateBinaryObjects(blz *BlobStor, f func(data []byte, blzID *blobovnicza.ID) error) error {
// Errors related to object reading and unmarshaling are logged and skipped.
func IterateBinaryObjects(blz *BlobStor, f func(addr oid.Address, data []byte, blzID *blobovnicza.ID) error) error {
var prm IteratePrm
prm.SetIterationHandler(func(elem IterationElement) error {
return f(elem.ObjectData(), elem.BlobovniczaID())
return f(elem.Address(), elem.ObjectData(), elem.BlobovniczaID())
})
prm.IgnoreErrors()
prm.SetErrorHandler(func(addr oid.Address, err error) error {
blz.log.Warn("error occurred during the iteration",
zap.Stringer("address", addr),
zap.String("err", err.Error()))
return nil
})
_, err := blz.Iterate(prm)
return err
}
// IterateObjects is a helper function which iterates over BlobStor and passes decoded objects to f.
func IterateObjects(blz *BlobStor, f func(obj *object.Object, blzID *blobovnicza.ID) error) error {
var obj *object.Object
return IterateBinaryObjects(blz, func(data []byte, blzID *blobovnicza.ID) error {
if obj == nil {
obj = object.New()
}
if err := obj.Unmarshal(data); err != nil {
return fmt.Errorf("could not unmarshal the object: %w", err)
}
return f(obj, blzID)
})
}

2
pkg/local_object_storage/blobstor/iterate_test.go
View file

@ -76,7 +76,7 @@ func TestIterateObjects(t *testing.T) {
require.NoError(t, err)
}
err := IterateBinaryObjects(blobStor, func(data []byte, blzID *blobovnicza.ID) error {
err := IterateBinaryObjects(blobStor, func(_ oid.Address, data []byte, blzID *blobovnicza.ID) error {
v, ok := mObjs[string(data)]
require.True(t, ok)