forked from TrueCloudLab/frostfs-node
[#541] blobstor/fstree: fix a bug in Iterate()
Be able to recover address from the path. Also add tests. Signed-off-by: Evgenii Stratonikov <evgeniy@nspcc.ru>
This commit is contained in:
parent
1b51dcf8a4
commit
213bbcbf2b
2 changed files with 171 additions and 7 deletions
|
@ -2,13 +2,13 @@ package fstree
|
|||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"github.com/nspcc-dev/neofs-api-go/pkg/container"
|
||||
objectSDK "github.com/nspcc-dev/neofs-api-go/pkg/object"
|
||||
)
|
||||
|
||||
|
@ -40,9 +40,30 @@ const (
|
|||
var ErrFileNotFound = errors.New("file not found")
|
||||
|
||||
func stringifyAddress(addr *objectSDK.Address) string {
|
||||
h := sha256.Sum256([]byte(addr.String()))
|
||||
return addr.ObjectID().String() + "." + addr.ContainerID().String()
|
||||
}
|
||||
|
||||
return hex.EncodeToString(h[:])
|
||||
func addressFromString(s string) (*objectSDK.Address, error) {
|
||||
ss := strings.SplitN(s, ".", 2)
|
||||
if len(ss) != 2 {
|
||||
return nil, errors.New("invalid address")
|
||||
}
|
||||
|
||||
oid := objectSDK.NewID()
|
||||
if err := oid.Parse(ss[0]); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cid := container.NewID()
|
||||
if err := cid.Parse(ss[1]); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
addr := objectSDK.NewAddress()
|
||||
addr.SetObjectID(oid)
|
||||
addr.SetContainerID(cid)
|
||||
|
||||
return addr, nil
|
||||
}
|
||||
|
||||
// Iterate iterates over all stored objects.
|
||||
|
@ -52,7 +73,7 @@ func (t *FSTree) Iterate(f func(addr *objectSDK.Address, data []byte) error) err
|
|||
|
||||
func (t *FSTree) iterate(depth int, curPath []string, f func(*objectSDK.Address, []byte) error) error {
|
||||
curName := strings.Join(curPath[1:], "")
|
||||
des, err := ioutil.ReadDir(curName)
|
||||
des, err := ioutil.ReadDir(path.Join(curPath...))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
@ -71,13 +92,15 @@ func (t *FSTree) iterate(depth int, curPath []string, f func(*objectSDK.Address,
|
|||
}
|
||||
}
|
||||
|
||||
addr := objectSDK.NewAddress()
|
||||
err := addr.Parse(curName + des[i].Name())
|
||||
if depth != t.Depth {
|
||||
continue
|
||||
}
|
||||
|
||||
addr, err := addressFromString(curName + des[i].Name())
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
curPath = append(curPath, des[i].Name())
|
||||
data, err := ioutil.ReadFile(path.Join(curPath...))
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
141
pkg/local_object_storage/blobstor/fstree/fstree_test.go
Normal file
141
pkg/local_object_storage/blobstor/fstree/fstree_test.go
Normal file
|
@ -0,0 +1,141 @@
|
|||
package fstree
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"os"
|
||||
"path"
|
||||
"testing"
|
||||
|
||||
"github.com/nspcc-dev/neofs-api-go/pkg/container"
|
||||
objectSDK "github.com/nspcc-dev/neofs-api-go/pkg/object"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func testCID() *container.ID {
|
||||
cs := [sha256.Size]byte{}
|
||||
_, _ = rand.Read(cs[:])
|
||||
|
||||
id := container.NewID()
|
||||
id.SetSHA256(cs)
|
||||
|
||||
return id
|
||||
}
|
||||
|
||||
func testOID() *objectSDK.ID {
|
||||
cs := [sha256.Size]byte{}
|
||||
_, _ = rand.Read(cs[:])
|
||||
|
||||
id := objectSDK.NewID()
|
||||
id.SetSHA256(cs)
|
||||
|
||||
return id
|
||||
}
|
||||
|
||||
func testAddress() *objectSDK.Address {
|
||||
a := objectSDK.NewAddress()
|
||||
a.SetObjectID(testOID())
|
||||
a.SetContainerID(testCID())
|
||||
|
||||
return a
|
||||
}
|
||||
|
||||
func TestAddressToString(t *testing.T) {
|
||||
addr := testAddress()
|
||||
s := stringifyAddress(addr)
|
||||
actual, err := addressFromString(s)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, addr, actual)
|
||||
}
|
||||
|
||||
func TestFSTree(t *testing.T) {
|
||||
tmpDir := path.Join(os.TempDir(), "neofs.fstree.test")
|
||||
require.NoError(t, os.Mkdir(tmpDir, os.ModePerm))
|
||||
t.Cleanup(func() { require.NoError(t, os.RemoveAll(tmpDir)) })
|
||||
|
||||
fs := FSTree{
|
||||
Info: Info{
|
||||
Permissions: os.ModePerm,
|
||||
RootPath: tmpDir,
|
||||
},
|
||||
Depth: 2,
|
||||
DirNameLen: 2,
|
||||
}
|
||||
|
||||
const count = 3
|
||||
var addrs []*objectSDK.Address
|
||||
|
||||
store := map[string][]byte{}
|
||||
|
||||
for i := 0; i < count; i++ {
|
||||
a := testAddress()
|
||||
addrs = append(addrs, a)
|
||||
|
||||
data := make([]byte, 10)
|
||||
_, _ = rand.Read(data[:])
|
||||
require.NoError(t, fs.Put(a, data))
|
||||
store[a.String()] = data
|
||||
}
|
||||
|
||||
t.Run("get", func(t *testing.T) {
|
||||
for _, a := range addrs {
|
||||
actual, err := fs.Get(a)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, store[a.String()], actual)
|
||||
}
|
||||
|
||||
_, err := fs.Get(testAddress())
|
||||
require.Error(t, err)
|
||||
})
|
||||
|
||||
t.Run("exists", func(t *testing.T) {
|
||||
for _, a := range addrs {
|
||||
_, err := fs.Exists(a)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
_, err := fs.Exists(testAddress())
|
||||
require.Error(t, err)
|
||||
})
|
||||
|
||||
t.Run("iterate", func(t *testing.T) {
|
||||
n := 0
|
||||
err := fs.Iterate(func(addr *objectSDK.Address, data []byte) error {
|
||||
n++
|
||||
expected, ok := store[addr.String()]
|
||||
require.True(t, ok, "object %s was not found", addr.String())
|
||||
require.Equal(t, data, expected)
|
||||
return nil
|
||||
})
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, count, n)
|
||||
|
||||
t.Run("leave early", func(t *testing.T) {
|
||||
n := 0
|
||||
errStop := errors.New("stop")
|
||||
err := fs.Iterate(func(addr *objectSDK.Address, data []byte) error {
|
||||
if n++; n == count-1 {
|
||||
return errStop
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
require.True(t, errors.Is(err, errStop))
|
||||
require.Equal(t, count-1, n)
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("delete", func(t *testing.T) {
|
||||
require.NoError(t, fs.Delete(addrs[0]))
|
||||
|
||||
_, err := fs.Exists(addrs[0])
|
||||
require.Error(t, err)
|
||||
|
||||
_, err = fs.Exists(addrs[1])
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Error(t, fs.Delete(testAddress()))
|
||||
})
|
||||
}
|
Loading…
Reference in a new issue