From 213bbcbf2bef59d3acc17abc6099eac3dd766174 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 19 May 2021 17:53:51 +0300 Subject: [PATCH] [#541] blobstor/fstree: fix a bug in `Iterate()` Be able to recover address from the path. Also add tests. Signed-off-by: Evgenii Stratonikov --- .../blobstor/fstree/fstree.go | 37 ++++- .../blobstor/fstree/fstree_test.go | 141 ++++++++++++++++++ 2 files changed, 171 insertions(+), 7 deletions(-) create mode 100644 pkg/local_object_storage/blobstor/fstree/fstree_test.go diff --git a/pkg/local_object_storage/blobstor/fstree/fstree.go b/pkg/local_object_storage/blobstor/fstree/fstree.go index 41da5e8f..3ee12581 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree.go @@ -2,13 +2,13 @@ package fstree import ( "crypto/sha256" - "encoding/hex" "errors" "io/ioutil" "os" "path" "strings" + "github.com/nspcc-dev/neofs-api-go/pkg/container" objectSDK "github.com/nspcc-dev/neofs-api-go/pkg/object" ) @@ -40,9 +40,30 @@ const ( var ErrFileNotFound = errors.New("file not found") func stringifyAddress(addr *objectSDK.Address) string { - h := sha256.Sum256([]byte(addr.String())) + return addr.ObjectID().String() + "." + addr.ContainerID().String() +} - return hex.EncodeToString(h[:]) +func addressFromString(s string) (*objectSDK.Address, error) { + ss := strings.SplitN(s, ".", 2) + if len(ss) != 2 { + return nil, errors.New("invalid address") + } + + oid := objectSDK.NewID() + if err := oid.Parse(ss[0]); err != nil { + return nil, err + } + + cid := container.NewID() + if err := cid.Parse(ss[1]); err != nil { + return nil, err + } + + addr := objectSDK.NewAddress() + addr.SetObjectID(oid) + addr.SetContainerID(cid) + + return addr, nil } // Iterate iterates over all stored objects. @@ -52,7 +73,7 @@ func (t *FSTree) Iterate(f func(addr *objectSDK.Address, data []byte) error) err func (t *FSTree) iterate(depth int, curPath []string, f func(*objectSDK.Address, []byte) error) error { curName := strings.Join(curPath[1:], "") - des, err := ioutil.ReadDir(curName) + des, err := ioutil.ReadDir(path.Join(curPath...)) if err != nil { return err } @@ -71,13 +92,15 @@ func (t *FSTree) iterate(depth int, curPath []string, f func(*objectSDK.Address, } } - addr := objectSDK.NewAddress() - err := addr.Parse(curName + des[i].Name()) + if depth != t.Depth { + continue + } + + addr, err := addressFromString(curName + des[i].Name()) if err != nil { continue } - curPath = append(curPath, des[i].Name()) data, err := ioutil.ReadFile(path.Join(curPath...)) if err != nil { return err diff --git a/pkg/local_object_storage/blobstor/fstree/fstree_test.go b/pkg/local_object_storage/blobstor/fstree/fstree_test.go new file mode 100644 index 00000000..2b87f529 --- /dev/null +++ b/pkg/local_object_storage/blobstor/fstree/fstree_test.go @@ -0,0 +1,141 @@ +package fstree + +import ( + "crypto/rand" + "crypto/sha256" + "errors" + "os" + "path" + "testing" + + "github.com/nspcc-dev/neofs-api-go/pkg/container" + objectSDK "github.com/nspcc-dev/neofs-api-go/pkg/object" + "github.com/stretchr/testify/require" +) + +func testCID() *container.ID { + cs := [sha256.Size]byte{} + _, _ = rand.Read(cs[:]) + + id := container.NewID() + id.SetSHA256(cs) + + return id +} + +func testOID() *objectSDK.ID { + cs := [sha256.Size]byte{} + _, _ = rand.Read(cs[:]) + + id := objectSDK.NewID() + id.SetSHA256(cs) + + return id +} + +func testAddress() *objectSDK.Address { + a := objectSDK.NewAddress() + a.SetObjectID(testOID()) + a.SetContainerID(testCID()) + + return a +} + +func TestAddressToString(t *testing.T) { + addr := testAddress() + s := stringifyAddress(addr) + actual, err := addressFromString(s) + require.NoError(t, err) + require.Equal(t, addr, actual) +} + +func TestFSTree(t *testing.T) { + tmpDir := path.Join(os.TempDir(), "neofs.fstree.test") + require.NoError(t, os.Mkdir(tmpDir, os.ModePerm)) + t.Cleanup(func() { require.NoError(t, os.RemoveAll(tmpDir)) }) + + fs := FSTree{ + Info: Info{ + Permissions: os.ModePerm, + RootPath: tmpDir, + }, + Depth: 2, + DirNameLen: 2, + } + + const count = 3 + var addrs []*objectSDK.Address + + store := map[string][]byte{} + + for i := 0; i < count; i++ { + a := testAddress() + addrs = append(addrs, a) + + data := make([]byte, 10) + _, _ = rand.Read(data[:]) + require.NoError(t, fs.Put(a, data)) + store[a.String()] = data + } + + t.Run("get", func(t *testing.T) { + for _, a := range addrs { + actual, err := fs.Get(a) + require.NoError(t, err) + require.Equal(t, store[a.String()], actual) + } + + _, err := fs.Get(testAddress()) + require.Error(t, err) + }) + + t.Run("exists", func(t *testing.T) { + for _, a := range addrs { + _, err := fs.Exists(a) + require.NoError(t, err) + } + + _, err := fs.Exists(testAddress()) + require.Error(t, err) + }) + + t.Run("iterate", func(t *testing.T) { + n := 0 + err := fs.Iterate(func(addr *objectSDK.Address, data []byte) error { + n++ + expected, ok := store[addr.String()] + require.True(t, ok, "object %s was not found", addr.String()) + require.Equal(t, data, expected) + return nil + }) + + require.NoError(t, err) + require.Equal(t, count, n) + + t.Run("leave early", func(t *testing.T) { + n := 0 + errStop := errors.New("stop") + err := fs.Iterate(func(addr *objectSDK.Address, data []byte) error { + if n++; n == count-1 { + return errStop + } + return nil + }) + + require.True(t, errors.Is(err, errStop)) + require.Equal(t, count-1, n) + }) + }) + + t.Run("delete", func(t *testing.T) { + require.NoError(t, fs.Delete(addrs[0])) + + _, err := fs.Exists(addrs[0]) + require.Error(t, err) + + _, err = fs.Exists(addrs[1]) + require.NoError(t, err) + + require.Error(t, fs.Delete(testAddress())) + }) +}