TrueCloudLab/frostfs-node
20
1
Fork 27
Code Issues 95 Pull requests 4 Projects Releases 1 Activity Actions

[#1085] fstree: allow to ignore errors during iteration

Browse source 
Signed-off-by: Evgenii Stratonikov <evgeniy@nspcc.ru>
This commit is contained in:
Evgenii Stratonikov 2022-01-20 18:53:29 +03:00 committed by LeL
parent e53ad2f468
commit c954f0e71b
5 changed files with 87 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

39
pkg/local_object_storage/blobstor/fstree/fstree.go
View file

@ -69,15 +69,36 @@ func addressFromString(s string) (*objectSDK.Address, error) {
return addr, nil
}
// Iterate iterates over all stored objects.
func (t *FSTree) Iterate(f func(addr *objectSDK.Address, data []byte) error) error {
return t.iterate(0, []string{t.RootPath}, f)
// IterationPrm contains iteraction parameters.
type IterationPrm struct {
handler func(addr *objectSDK.Address, data []byte) error
ignoreErrors bool
}
func (t *FSTree) iterate(depth int, curPath []string, f func(*objectSDK.Address, []byte) error) error {
// WithHandler sets a function to call on each object.
func (p *IterationPrm) WithHandler(f func(addr *objectSDK.Address, data []byte) error) *IterationPrm {
p.handler = f
return p
}
// WithIgnoreErrors sets a flag indicating whether errors should be ignored.
func (p *IterationPrm) WithIgnoreErrors(ignore bool) *IterationPrm {
p.ignoreErrors = ignore
return p
}
// Iterate iterates over all stored objects.
func (t *FSTree) Iterate(prm *IterationPrm) error {
return t.iterate(0, []string{t.RootPath}, prm)
}
func (t *FSTree) iterate(depth int, curPath []string, prm *IterationPrm) error {
curName := strings.Join(curPath[1:], "")
des, err := os.ReadDir(path.Join(curPath...))
if err != nil {
if prm.ignoreErrors {
return nil
}
return err
}
@ -89,8 +110,10 @@ func (t *FSTree) iterate(depth int, curPath []string, f func(*objectSDK.Address,
curPath[l] = des[i].Name()
if !isLast && des[i].IsDir() {
err := t.iterate(depth+1, curPath, f)
err := t.iterate(depth+1, curPath, prm)
if err != nil {
// Must be error from handler in case errors are ignored.
// Need to report.
return err
}
}
@ -106,10 +129,14 @@ func (t *FSTree) iterate(depth int, curPath []string, f func(*objectSDK.Address,
data, err := os.ReadFile(path.Join(curPath...))
if err != nil {
if prm.ignoreErrors {
continue
}
return err
}
if err := f(addr, data); err != nil {
if err := prm.handler(addr, data); err != nil {
// Error occurred in handler, outside of our scope, needs to be reported.
return err
}
}

49
pkg/local_object_storage/blobstor/fstree/fstree_test.go
View file

@ -6,10 +6,13 @@ import (
"errors"
"os"
"path"
"path/filepath"
"testing"
"github.com/nspcc-dev/neofs-node/pkg/util"
cidtest "github.com/nspcc-dev/neofs-sdk-go/container/id/test"
objectSDK "github.com/nspcc-dev/neofs-sdk-go/object"
objecttest "github.com/nspcc-dev/neofs-sdk-go/object/test"
"github.com/stretchr/testify/require"
)
@ -91,13 +94,13 @@ func TestFSTree(t *testing.T) {
t.Run("iterate", func(t *testing.T) {
n := 0
err := fs.Iterate(func(addr *objectSDK.Address, data []byte) error {
err := fs.Iterate(new(IterationPrm).WithHandler(func(addr *objectSDK.Address, data []byte) error {
n++
expected, ok := store[addr.String()]
require.True(t, ok, "object %s was not found", addr.String())
require.Equal(t, data, expected)
return nil
})
}))
require.NoError(t, err)
require.Equal(t, count, n)
@ -105,16 +108,54 @@ func TestFSTree(t *testing.T) {
t.Run("leave early", func(t *testing.T) {
n := 0
errStop := errors.New("stop")
err := fs.Iterate(func(addr *objectSDK.Address, data []byte) error {
err := fs.Iterate(new(IterationPrm).WithHandler(func(addr *objectSDK.Address, data []byte) error {
if n++; n == count-1 {
return errStop
}
return nil
})
}))
require.True(t, errors.Is(err, errStop))
require.Equal(t, count-1, n)
})
t.Run("ignore errors", func(t *testing.T) {
n := 0
// Unreadable directory.
require.NoError(t, os.Mkdir(filepath.Join(fs.RootPath, "ZZ"), 0))
// Unreadable file.
p := fs.treePath(objecttest.Address())
require.NoError(t, util.MkdirAllX(filepath.Dir(p), fs.Permissions))
require.NoError(t, os.WriteFile(p, []byte{1, 2, 3}, 0))
// Invalid address.
p = fs.treePath(objecttest.Address()) + ".invalid"
require.NoError(t, util.MkdirAllX(filepath.Dir(p), fs.Permissions))
require.NoError(t, os.WriteFile(p, []byte{1, 2, 3}, fs.Permissions))
err := fs.Iterate(new(IterationPrm).WithHandler(func(addr *objectSDK.Address, data []byte) error {
n++
return nil
}).WithIgnoreErrors(true))
require.NoError(t, err)
require.Equal(t, count, n)
t.Run("error from handler is returned", func(t *testing.T) {
expectedErr := errors.New("expected error")
n := 0
err := fs.Iterate(new(IterationPrm).WithHandler(func(addr *objectSDK.Address, data []byte) error {
n++
if n == count/2 { // process some iterations
return expectedErr
}
return nil
}).WithIgnoreErrors(true))
require.True(t, errors.Is(err, expectedErr), "got: %v")
require.Equal(t, count/2, n)
})
})
})
t.Run("delete", func(t *testing.T) {

5
pkg/local_object_storage/blobstor/iterate.go
View file

@ -5,6 +5,7 @@ import (
"github.com/nspcc-dev/neofs-node/pkg/core/object"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree"
objectSDK "github.com/nspcc-dev/neofs-sdk-go/object"
)
@ -78,7 +79,7 @@ func (b *BlobStor) Iterate(prm IteratePrm) (*IterateRes, error) {
elem.blzID = nil
err = b.fsTree.Iterate(func(_ *objectSDK.Address, data []byte) error {
err = b.fsTree.Iterate(new(fstree.IterationPrm).WithHandler(func(_ *objectSDK.Address, data []byte) error {
// decompress the data
elem.data, err = b.decompressor(data)
if err != nil {
@ -86,7 +87,7 @@ func (b *BlobStor) Iterate(prm IteratePrm) (*IterateRes, error) {
}
return prm.handler(elem)
})
}))
if err != nil {
return nil, fmt.Errorf("fs tree iterator failure: %w", err)
}

5
pkg/local_object_storage/writecache/flush.go
View file

@ -8,6 +8,7 @@ import (
"github.com/nspcc-dev/neofs-node/pkg/core/object"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree"
meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
objectSDK "github.com/nspcc-dev/neofs-sdk-go/object"
"go.etcd.io/bbolt"
@ -132,7 +133,7 @@ func (c *cache) flushBigObjects() {
}
evictNum := 0
_ = c.fsTree.Iterate(func(addr *objectSDK.Address, data []byte) error {
_ = c.fsTree.Iterate(new(fstree.IterationPrm).WithHandler(func(addr *objectSDK.Address, data []byte) error {
sAddr := addr.String()
if _, ok := c.store.flushed.Peek(sAddr); ok {
@ -160,7 +161,7 @@ func (c *cache) flushBigObjects() {
evictNum++
return nil
})
}))
// evict objects which were successfully written to BlobStor
c.evictObjects(evictNum)

5
pkg/local_object_storage/writecache/iterate.go
View file

@ -4,6 +4,7 @@ import (
"errors"
"fmt"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree"
"github.com/nspcc-dev/neofs-sdk-go/object"
"go.etcd.io/bbolt"
)
@ -34,12 +35,12 @@ func (c *cache) Iterate(f func([]byte) error) error {
return err
}
return c.fsTree.Iterate(func(addr *object.Address, data []byte) error {
return c.fsTree.Iterate(new(fstree.IterationPrm).WithHandler(func(addr *object.Address, data []byte) error {
if _, ok := c.flushed.Peek(addr.String()); ok {
return nil
}
return f(data)
})
}))
}
// IterateDB iterates over all objects stored in bbolt.DB instance and passes them to f until error return.