[#541] blobstor/fstree: fix a bug in Iterate()

Be able to recover address from the path. Also add tests.

Signed-off-by: Evgenii Stratonikov <evgeniy@nspcc.ru>
This commit is contained in:
Evgenii Stratonikov 2021-05-19 17:53:51 +03:00 committed by Alex Vanin
parent 1b51dcf8a4
commit 213bbcbf2b
2 changed files with 171 additions and 7 deletions

View file

@ -2,13 +2,13 @@ package fstree
import ( import (
"crypto/sha256" "crypto/sha256"
"encoding/hex"
"errors" "errors"
"io/ioutil" "io/ioutil"
"os" "os"
"path" "path"
"strings" "strings"
"github.com/nspcc-dev/neofs-api-go/pkg/container"
objectSDK "github.com/nspcc-dev/neofs-api-go/pkg/object" objectSDK "github.com/nspcc-dev/neofs-api-go/pkg/object"
) )
@ -40,9 +40,30 @@ const (
var ErrFileNotFound = errors.New("file not found") var ErrFileNotFound = errors.New("file not found")
func stringifyAddress(addr *objectSDK.Address) string { func stringifyAddress(addr *objectSDK.Address) string {
h := sha256.Sum256([]byte(addr.String())) return addr.ObjectID().String() + "." + addr.ContainerID().String()
}
return hex.EncodeToString(h[:]) func addressFromString(s string) (*objectSDK.Address, error) {
ss := strings.SplitN(s, ".", 2)
if len(ss) != 2 {
return nil, errors.New("invalid address")
}
oid := objectSDK.NewID()
if err := oid.Parse(ss[0]); err != nil {
return nil, err
}
cid := container.NewID()
if err := cid.Parse(ss[1]); err != nil {
return nil, err
}
addr := objectSDK.NewAddress()
addr.SetObjectID(oid)
addr.SetContainerID(cid)
return addr, nil
} }
// Iterate iterates over all stored objects. // Iterate iterates over all stored objects.
@ -52,7 +73,7 @@ func (t *FSTree) Iterate(f func(addr *objectSDK.Address, data []byte) error) err
func (t *FSTree) iterate(depth int, curPath []string, f func(*objectSDK.Address, []byte) error) error { func (t *FSTree) iterate(depth int, curPath []string, f func(*objectSDK.Address, []byte) error) error {
curName := strings.Join(curPath[1:], "") curName := strings.Join(curPath[1:], "")
des, err := ioutil.ReadDir(curName) des, err := ioutil.ReadDir(path.Join(curPath...))
if err != nil { if err != nil {
return err return err
} }
@ -71,13 +92,15 @@ func (t *FSTree) iterate(depth int, curPath []string, f func(*objectSDK.Address,
} }
} }
addr := objectSDK.NewAddress() if depth != t.Depth {
err := addr.Parse(curName + des[i].Name()) continue
}
addr, err := addressFromString(curName + des[i].Name())
if err != nil { if err != nil {
continue continue
} }
curPath = append(curPath, des[i].Name())
data, err := ioutil.ReadFile(path.Join(curPath...)) data, err := ioutil.ReadFile(path.Join(curPath...))
if err != nil { if err != nil {
return err return err

View file

@ -0,0 +1,141 @@
package fstree
import (
"crypto/rand"
"crypto/sha256"
"errors"
"os"
"path"
"testing"
"github.com/nspcc-dev/neofs-api-go/pkg/container"
objectSDK "github.com/nspcc-dev/neofs-api-go/pkg/object"
"github.com/stretchr/testify/require"
)
func testCID() *container.ID {
cs := [sha256.Size]byte{}
_, _ = rand.Read(cs[:])
id := container.NewID()
id.SetSHA256(cs)
return id
}
func testOID() *objectSDK.ID {
cs := [sha256.Size]byte{}
_, _ = rand.Read(cs[:])
id := objectSDK.NewID()
id.SetSHA256(cs)
return id
}
func testAddress() *objectSDK.Address {
a := objectSDK.NewAddress()
a.SetObjectID(testOID())
a.SetContainerID(testCID())
return a
}
func TestAddressToString(t *testing.T) {
addr := testAddress()
s := stringifyAddress(addr)
actual, err := addressFromString(s)
require.NoError(t, err)
require.Equal(t, addr, actual)
}
func TestFSTree(t *testing.T) {
tmpDir := path.Join(os.TempDir(), "neofs.fstree.test")
require.NoError(t, os.Mkdir(tmpDir, os.ModePerm))
t.Cleanup(func() { require.NoError(t, os.RemoveAll(tmpDir)) })
fs := FSTree{
Info: Info{
Permissions: os.ModePerm,
RootPath: tmpDir,
},
Depth: 2,
DirNameLen: 2,
}
const count = 3
var addrs []*objectSDK.Address
store := map[string][]byte{}
for i := 0; i < count; i++ {
a := testAddress()
addrs = append(addrs, a)
data := make([]byte, 10)
_, _ = rand.Read(data[:])
require.NoError(t, fs.Put(a, data))
store[a.String()] = data
}
t.Run("get", func(t *testing.T) {
for _, a := range addrs {
actual, err := fs.Get(a)
require.NoError(t, err)
require.Equal(t, store[a.String()], actual)
}
_, err := fs.Get(testAddress())
require.Error(t, err)
})
t.Run("exists", func(t *testing.T) {
for _, a := range addrs {
_, err := fs.Exists(a)
require.NoError(t, err)
}
_, err := fs.Exists(testAddress())
require.Error(t, err)
})
t.Run("iterate", func(t *testing.T) {
n := 0
err := fs.Iterate(func(addr *objectSDK.Address, data []byte) error {
n++
expected, ok := store[addr.String()]
require.True(t, ok, "object %s was not found", addr.String())
require.Equal(t, data, expected)
return nil
})
require.NoError(t, err)
require.Equal(t, count, n)
t.Run("leave early", func(t *testing.T) {
n := 0
errStop := errors.New("stop")
err := fs.Iterate(func(addr *objectSDK.Address, data []byte) error {
if n++; n == count-1 {
return errStop
}
return nil
})
require.True(t, errors.Is(err, errStop))
require.Equal(t, count-1, n)
})
})
t.Run("delete", func(t *testing.T) {
require.NoError(t, fs.Delete(addrs[0]))
_, err := fs.Exists(addrs[0])
require.Error(t, err)
_, err = fs.Exists(addrs[1])
require.NoError(t, err)
require.Error(t, fs.Delete(testAddress()))
})
}