realloc/frostfs-node
1
0
Fork 0
forked from TrueCloudLab/frostfs-node
Code Releases Packages Activity

[#1131] *: Adopt SDK changes

Browse source 
`object.Address` has been moved to `object/address`
`object.ID` has been moved to `object/id`

Signed-off-by: Pavel Karpy <carpawell@nspcc.ru>
This commit is contained in:
Pavel Karpy 2022-01-26 15:11:13 +03:00 • committed by Alex Vanin
parent 1c821d6c36
commit 1667ec9e6d
139 changed files with 652 additions and 562 deletions
Download patch file Download diff file Expand all files Collapse all files

12
pkg/local_object_storage/blobstor/blobovnicza.go
View file

@ -13,7 +13,7 @@ import (
"github.com/nspcc-dev/neofs-node/pkg/core/object"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza"
storagelog "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/internal/log"
objectSDK "github.com/nspcc-dev/neofs-sdk-go/object"
addressSDK "github.com/nspcc-dev/neofs-sdk-go/object/address"
"go.uber.org/zap"
)
@ -132,7 +132,7 @@ func indexSlice(number uint64) []uint64 {
// save object in the maximum weight blobobnicza.
//
// returns error if could not save object in any blobovnicza.
func (b *blobovniczas) put(addr *objectSDK.Address, data []byte) (*blobovnicza.ID, error) {
func (b *blobovniczas) put(addr *addressSDK.Address, data []byte) (*blobovnicza.ID, error) {
prm := new(blobovnicza.PutPrm)
prm.SetAddress(addr)
prm.SetMarshaledObject(data)
@ -648,7 +648,7 @@ func (b *blobovniczas) iterateBlobovniczas(ignoreErrors bool, f func(string, *bl
}
// iterator over the paths of blobovniczas sorted by weight.
func (b *blobovniczas) iterateSortedLeaves(addr *objectSDK.Address, f func(string) (bool, error)) error {
func (b *blobovniczas) iterateSortedLeaves(addr *addressSDK.Address, f func(string) (bool, error)) error {
_, err := b.iterateSorted(
addr,
make([]string, 0, b.blzShallowDepth),
@ -660,7 +660,7 @@ func (b *blobovniczas) iterateSortedLeaves(addr *objectSDK.Address, f func(strin
}
// iterator over directories with blobovniczas sorted by weight.
func (b *blobovniczas) iterateDeepest(addr *objectSDK.Address, f func(string) (bool, error)) error {
func (b *blobovniczas) iterateDeepest(addr *addressSDK.Address, f func(string) (bool, error)) error {
depth := b.blzShallowDepth
if depth > 0 {
depth--
@ -677,7 +677,7 @@ func (b *blobovniczas) iterateDeepest(addr *objectSDK.Address, f func(string) (b
}
// iterator over particular level of directories.
func (b *blobovniczas) iterateSorted(addr *objectSDK.Address, curPath []string, execDepth uint64, f func([]string) (bool, error)) (bool, error) {
func (b *blobovniczas) iterateSorted(addr *addressSDK.Address, curPath []string, execDepth uint64, f func([]string) (bool, error)) (bool, error) {
indices := indexSlice(b.blzShallowWidth)
hrw.SortSliceByValue(indices, addressHash(addr, filepath.Join(curPath...)))
@ -906,7 +906,7 @@ func (b *blobovniczas) openBlobovnicza(p string) (*blobovnicza.Blobovnicza, erro
}
// returns hash of the object address.
func addressHash(addr *objectSDK.Address, path string) uint64 {
func addressHash(addr *addressSDK.Address, path string) uint64 {
var a string
if addr != nil {

10
pkg/local_object_storage/blobstor/blobovnicza_test.go
View file

@ -11,6 +11,8 @@ import (
"github.com/nspcc-dev/neofs-node/pkg/util/logger/test"
cidtest "github.com/nspcc-dev/neofs-sdk-go/container/id/test"
objectSDK "github.com/nspcc-dev/neofs-sdk-go/object"
addressSDK "github.com/nspcc-dev/neofs-sdk-go/object/address"
oidSDK "github.com/nspcc-dev/neofs-sdk-go/object/id"
"github.com/stretchr/testify/require"
)
@ -20,11 +22,11 @@ func testSHA256() (h [sha256.Size]byte) {
return h
}
func testAddress() *objectSDK.Address {
oid := objectSDK.NewID()
func testAddress() *addressSDK.Address {
oid := oidSDK.NewID()
oid.SetSHA256(testSHA256())
addr := objectSDK.NewAddress()
addr := addressSDK.NewAddress()
addr.SetObjectID(oid)
addr.SetContainerID(cidtest.ID())
@ -84,7 +86,7 @@ func TestBlobovniczas(t *testing.T) {
objSz := uint64(szLim / 2)
addrList := make([]*objectSDK.Address, 0)
addrList := make([]*addressSDK.Address, 0)
minFitObjNum := width * depth * szLim / objSz
for i := uint64(0); i < minFitObjNum; i++ {

6
pkg/local_object_storage/blobstor/exists.go
View file

@ -4,7 +4,7 @@ import (
"errors"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree"
"github.com/nspcc-dev/neofs-sdk-go/object"
addressSDK "github.com/nspcc-dev/neofs-sdk-go/object/address"
)
// ExistsPrm groups the parameters of Exists operation.
@ -46,7 +46,7 @@ func (b *BlobStor) Exists(prm *ExistsPrm) (*ExistsRes, error) {
}
// checks if object is presented in shallow dir.
func (b *BlobStor) existsBig(addr *object.Address) (bool, error) {
func (b *BlobStor) existsBig(addr *addressSDK.Address) (bool, error) {
_, err := b.fsTree.Exists(addr)
if errors.Is(err, fstree.ErrFileNotFound) {
return false, nil
@ -56,7 +56,7 @@ func (b *BlobStor) existsBig(addr *object.Address) (bool, error) {
}
// checks if object is presented in blobovnicza.
func (b *BlobStor) existsSmall(addr *object.Address) (bool, error) {
func (b *BlobStor) existsSmall(_ *addressSDK.Address) (bool, error) {
// TODO: implement
return false, nil
}

25
pkg/local_object_storage/blobstor/fstree/fstree.go
View file

@ -11,7 +11,8 @@ import (
"github.com/nspcc-dev/neofs-node/pkg/util"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
objectSDK "github.com/nspcc-dev/neofs-sdk-go/object"
addressSDK "github.com/nspcc-dev/neofs-sdk-go/object/address"
oidSDK "github.com/nspcc-dev/neofs-sdk-go/object/id"
)
// FSTree represents object storage as filesystem tree.
@ -41,17 +42,17 @@ const (
// ErrFileNotFound is returned when file is missing.
var ErrFileNotFound = errors.New("file not found")
func stringifyAddress(addr *objectSDK.Address) string {
func stringifyAddress(addr *addressSDK.Address) string {
return addr.ObjectID().String() + "." + addr.ContainerID().String()
}
func addressFromString(s string) (*objectSDK.Address, error) {
func addressFromString(s string) (*addressSDK.Address, error) {
ss := strings.SplitN(s, ".", 2)
if len(ss) != 2 {
return nil, errors.New("invalid address")
}
oid := objectSDK.NewID()
oid := oidSDK.NewID()
if err := oid.Parse(ss[0]); err != nil {
return nil, err
}
@ -61,7 +62,7 @@ func addressFromString(s string) (*objectSDK.Address, error) {
return nil, err
}
addr := objectSDK.NewAddress()
addr := addressSDK.NewAddress()
addr.SetObjectID(oid)
addr.SetContainerID(id)
@ -70,12 +71,12 @@ func addressFromString(s string) (*objectSDK.Address, error) {
// IterationPrm contains iteraction parameters.
type IterationPrm struct {
handler func(addr *objectSDK.Address, data []byte) error
handler func(addr *addressSDK.Address, data []byte) error
ignoreErrors bool
}
// WithHandler sets a function to call on each object.
func (p *IterationPrm) WithHandler(f func(addr *objectSDK.Address, data []byte) error) *IterationPrm {
func (p *IterationPrm) WithHandler(f func(addr *addressSDK.Address, data []byte) error) *IterationPrm {
p.handler = f
return p
}
@ -143,7 +144,7 @@ func (t *FSTree) iterate(depth int, curPath []string, prm *IterationPrm) error {
return nil
}
func (t *FSTree) treePath(addr *objectSDK.Address) string {
func (t *FSTree) treePath(addr *addressSDK.Address) string {
sAddr := stringifyAddress(addr)
dirs := make([]string, 0, t.Depth+1+1) // 1 for root, 1 for file
@ -160,7 +161,7 @@ func (t *FSTree) treePath(addr *objectSDK.Address) string {
}
// Delete removes object with the specified address from storage.
func (t *FSTree) Delete(addr *objectSDK.Address) error {
func (t *FSTree) Delete(addr *addressSDK.Address) error {
p, err := t.Exists(addr)
if err != nil {
return err
@ -171,7 +172,7 @@ func (t *FSTree) Delete(addr *objectSDK.Address) error {
// Exists returns path to file with object contents if it exists in storage
// and an error otherwise.
func (t *FSTree) Exists(addr *objectSDK.Address) (string, error) {
func (t *FSTree) Exists(addr *addressSDK.Address) (string, error) {
p := t.treePath(addr)
_, err := os.Stat(p)
@ -183,7 +184,7 @@ func (t *FSTree) Exists(addr *objectSDK.Address) (string, error) {
}
// Put puts object in storage.
func (t *FSTree) Put(addr *objectSDK.Address, data []byte) error {
func (t *FSTree) Put(addr *addressSDK.Address, data []byte) error {
p := t.treePath(addr)
if err := util.MkdirAllX(filepath.Dir(p), t.Permissions); err != nil {
@ -194,7 +195,7 @@ func (t *FSTree) Put(addr *objectSDK.Address, data []byte) error {
}
// Get returns object from storage by address.
func (t *FSTree) Get(addr *objectSDK.Address) ([]byte, error) {
func (t *FSTree) Get(addr *addressSDK.Address) ([]byte, error) {
p := t.treePath(addr)
if _, err := os.Stat(p); os.IsNotExist(err) {

23
pkg/local_object_storage/blobstor/fstree/fstree_test.go
View file

@ -10,23 +10,24 @@ import (
"github.com/nspcc-dev/neofs-node/pkg/util"
cidtest "github.com/nspcc-dev/neofs-sdk-go/container/id/test"
objectSDK "github.com/nspcc-dev/neofs-sdk-go/object"
objecttest "github.com/nspcc-dev/neofs-sdk-go/object/test"
addressSDK "github.com/nspcc-dev/neofs-sdk-go/object/address"
objecttest "github.com/nspcc-dev/neofs-sdk-go/object/address/test"
oidSDK "github.com/nspcc-dev/neofs-sdk-go/object/id"
"github.com/stretchr/testify/require"
)
func testOID() *objectSDK.ID {
func testOID() *oidSDK.ID {
cs := [sha256.Size]byte{}
_, _ = rand.Read(cs[:])
id := objectSDK.NewID()
id := oidSDK.NewID()
id.SetSHA256(cs)
return id
}
func testAddress() *objectSDK.Address {
a := objectSDK.NewAddress()
func testAddress() *addressSDK.Address {
a := addressSDK.NewAddress()
a.SetObjectID(testOID())
a.SetContainerID(cidtest.ID())
@ -56,7 +57,7 @@ func TestFSTree(t *testing.T) {
}
const count = 3
var addrs []*objectSDK.Address
var addrs []*addressSDK.Address
store := map[string][]byte{}
@ -93,7 +94,7 @@ func TestFSTree(t *testing.T) {
t.Run("iterate", func(t *testing.T) {
n := 0
err := fs.Iterate(new(IterationPrm).WithHandler(func(addr *objectSDK.Address, data []byte) error {
err := fs.Iterate(new(IterationPrm).WithHandler(func(addr *addressSDK.Address, data []byte) error {
n++
expected, ok := store[addr.String()]
require.True(t, ok, "object %s was not found", addr.String())
@ -107,7 +108,7 @@ func TestFSTree(t *testing.T) {
t.Run("leave early", func(t *testing.T) {
n := 0
errStop := errors.New("stop")
err := fs.Iterate(new(IterationPrm).WithHandler(func(addr *objectSDK.Address, data []byte) error {
err := fs.Iterate(new(IterationPrm).WithHandler(func(addr *addressSDK.Address, data []byte) error {
if n++; n == count-1 {
return errStop
}
@ -134,7 +135,7 @@ func TestFSTree(t *testing.T) {
require.NoError(t, util.MkdirAllX(filepath.Dir(p), fs.Permissions))
require.NoError(t, os.WriteFile(p, []byte{1, 2, 3}, fs.Permissions))
err := fs.Iterate(new(IterationPrm).WithHandler(func(addr *objectSDK.Address, data []byte) error {
err := fs.Iterate(new(IterationPrm).WithHandler(func(addr *addressSDK.Address, data []byte) error {
n++
return nil
}).WithIgnoreErrors(true))
@ -144,7 +145,7 @@ func TestFSTree(t *testing.T) {
t.Run("error from handler is returned", func(t *testing.T) {
expectedErr := errors.New("expected error")
n := 0
err := fs.Iterate(new(IterationPrm).WithHandler(func(addr *objectSDK.Address, data []byte) error {
err := fs.Iterate(new(IterationPrm).WithHandler(func(addr *addressSDK.Address, data []byte) error {
n++
if n == count/2 { // process some iterations
return expectedErr

4
pkg/local_object_storage/blobstor/iterate.go
View file

@ -6,7 +6,7 @@ import (
"github.com/nspcc-dev/neofs-node/pkg/core/object"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree"
objectSDK "github.com/nspcc-dev/neofs-sdk-go/object"
addressSDK "github.com/nspcc-dev/neofs-sdk-go/object/address"
)
// IterationElement represents a unit of elements through which Iterate operation passes.
@ -88,7 +88,7 @@ func (b *BlobStor) Iterate(prm IteratePrm) (*IterateRes, error) {
elem.blzID = nil
err = b.fsTree.Iterate(new(fstree.IterationPrm).WithHandler(func(_ *objectSDK.Address, data []byte) error {
err = b.fsTree.Iterate(new(fstree.IterationPrm).WithHandler(func(_ *addressSDK.Address, data []byte) error {
// decompress the data
elem.data, err = b.decompressor(data)
if err != nil {

13
pkg/local_object_storage/blobstor/iterate_test.go
View file

@ -10,9 +10,10 @@ import (
"testing"
"github.com/klauspost/compress/zstd"
"github.com/nspcc-dev/neofs-node/pkg/core/object"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza"
"github.com/nspcc-dev/neofs-sdk-go/object"
objecttest "github.com/nspcc-dev/neofs-sdk-go/object/test"
addressSDK "github.com/nspcc-dev/neofs-sdk-go/object/address"
objecttest "github.com/nspcc-dev/neofs-sdk-go/object/address/test"
"github.com/stretchr/testify/require"
)
@ -44,7 +45,7 @@ func TestIterateObjects(t *testing.T) {
type addrData struct {
big bool
addr *object.Address
addr *addressSDK.Address
data []byte
}
@ -113,7 +114,7 @@ func TestIterate_IgnoreErrors(t *testing.T) {
require.NoError(t, bs.Open())
require.NoError(t, bs.Init())
addrs := make([]*object.Address, objCount)
addrs := make([]*addressSDK.Address, objCount)
for i := range addrs {
addrs[i] = objecttest.Address()
obj := object.NewRaw()
@ -170,7 +171,7 @@ func TestIterate_IgnoreErrors(t *testing.T) {
prm.IgnoreErrors()
t.Run("skip invalid objects", func(t *testing.T) {
actual := make([]*object.Address, 0, len(addrs))
actual := make([]*addressSDK.Address, 0, len(addrs))
prm.SetIterationHandler(func(e IterationElement) error {
obj := object.New()
err := obj.Unmarshal(e.data)
@ -178,7 +179,7 @@ func TestIterate_IgnoreErrors(t *testing.T) {
return err
}
addr := object.NewAddress()
addr := addressSDK.NewAddress()
addr.SetContainerID(obj.ContainerID())
addr.SetObjectID(obj.ID())
actual = append(actual, addr)

3
pkg/local_object_storage/blobstor/put.go
View file

@ -7,6 +7,7 @@ import (
"github.com/nspcc-dev/neofs-node/pkg/core/object"
storagelog "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/internal/log"
objectSDK "github.com/nspcc-dev/neofs-sdk-go/object"
addressSDK "github.com/nspcc-dev/neofs-sdk-go/object/address"
)
// PutPrm groups the parameters of Put operation.
@ -69,7 +70,7 @@ func (b *BlobStor) NeedsCompression(obj *object.Object) bool {
}
// PutRaw saves already marshaled object in BLOB storage.
func (b *BlobStor) PutRaw(addr *objectSDK.Address, data []byte, compress bool) (*PutRes, error) {
func (b *BlobStor) PutRaw(addr *addressSDK.Address, data []byte, compress bool) (*PutRes, error) {
big := b.isBig(data)
if compress {

5
pkg/local_object_storage/blobstor/util.go
View file

@ -4,14 +4,15 @@ import (
"github.com/nspcc-dev/neofs-node/pkg/core/object"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza"
objectSDK "github.com/nspcc-dev/neofs-sdk-go/object"
addressSDK "github.com/nspcc-dev/neofs-sdk-go/object/address"
)
type address struct {
addr *objectSDK.Address
addr *addressSDK.Address
}
// SetAddress sets the address of the requested object.
func (a *address) SetAddress(addr *objectSDK.Address) {
func (a *address) SetAddress(addr *addressSDK.Address) {
a.addr = addr
}