realloc/frostfs-node
1
0
Fork 0
forked from TrueCloudLab/frostfs-node
Code Releases Packages Activity

[#1454] Upgrade NeoFS SDK Go module with new IDs

Browse source 
Core changes:
 * avoid package-colliding variable naming
 * avoid using pointers to IDs where unnecessary
 * avoid using `idSDK` import alias pattern
 * use `EncodeToString` for protocol string calculation and `String` for
  printing

Signed-off-by: Leonard Lyubich <leonard@nspcc.ru>
This commit is contained in:
Leonard Lyubich 2022-05-31 20:00:41 +03:00 • committed by LeL
parent cc6209e8a0
commit 1c30414a6c
218 changed files with 2095 additions and 2521 deletions
Download patch file Download diff file Expand all files Collapse all files

22
pkg/local_object_storage/blobstor/blobovnicza.go
View file

@ -15,7 +15,7 @@ import (
storagelog "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/internal/log"
apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status"
objectSDK "github.com/nspcc-dev/neofs-sdk-go/object"
addressSDK "github.com/nspcc-dev/neofs-sdk-go/object/address"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"go.uber.org/zap"
)
@ -136,7 +136,7 @@ func indexSlice(number uint64) []uint64 {
// save object in the maximum weight blobobnicza.
//
// returns error if could not save object in any blobovnicza.
func (b *blobovniczas) put(addr *addressSDK.Address, data []byte) (*blobovnicza.ID, error) {
func (b *blobovniczas) put(addr oid.Address, data []byte) (*blobovnicza.ID, error) {
prm := new(blobovnicza.PutPrm)
prm.SetAddress(addr)
prm.SetMarshaledObject(data)
@ -220,7 +220,7 @@ func (b *blobovniczas) get(prm *GetSmallPrm) (res *GetSmallRes, err error) {
activeCache := make(map[string]struct{})
err = b.iterateSortedLeaves(prm.addr, func(p string) (bool, error) {
err = b.iterateSortedLeaves(&prm.addr, func(p string) (bool, error) {
dirPath := filepath.Dir(p)
_, ok := activeCache[dirPath]
@ -270,7 +270,7 @@ func (b *blobovniczas) delete(prm *DeleteSmallPrm) (res *DeleteSmallRes, err err
activeCache := make(map[string]struct{})
err = b.iterateSortedLeaves(prm.addr, func(p string) (bool, error) {
err = b.iterateSortedLeaves(&prm.addr, func(p string) (bool, error) {
dirPath := filepath.Dir(p)
// don't process active blobovnicza of the level twice
@ -322,7 +322,7 @@ func (b *blobovniczas) getRange(prm *GetRangeSmallPrm) (res *GetRangeSmallRes, e
activeCache := make(map[string]struct{})
err = b.iterateSortedLeaves(prm.addr, func(p string) (bool, error) {
err = b.iterateSortedLeaves(&prm.addr, func(p string) (bool, error) {
dirPath := filepath.Dir(p)
_, ok := activeCache[dirPath]
@ -665,7 +665,7 @@ func (b *blobovniczas) iterateBlobovniczas(ignoreErrors bool, f func(string, *bl
}
// iterator over the paths of blobovniczas sorted by weight.
func (b *blobovniczas) iterateSortedLeaves(addr *addressSDK.Address, f func(string) (bool, error)) error {
func (b *blobovniczas) iterateSortedLeaves(addr *oid.Address, f func(string) (bool, error)) error {
_, err := b.iterateSorted(
addr,
make([]string, 0, b.blzShallowDepth),
@ -677,14 +677,14 @@ func (b *blobovniczas) iterateSortedLeaves(addr *addressSDK.Address, f func(stri
}
// iterator over directories with blobovniczas sorted by weight.
func (b *blobovniczas) iterateDeepest(addr *addressSDK.Address, f func(string) (bool, error)) error {
func (b *blobovniczas) iterateDeepest(addr oid.Address, f func(string) (bool, error)) error {
depth := b.blzShallowDepth
if depth > 0 {
depth--
}
_, err := b.iterateSorted(
addr,
&addr,
make([]string, 0, depth),
depth,
func(p []string) (bool, error) { return f(filepath.Join(p...)) },
@ -694,7 +694,7 @@ func (b *blobovniczas) iterateDeepest(addr *addressSDK.Address, f func(string) (
}
// iterator over particular level of directories.
func (b *blobovniczas) iterateSorted(addr *addressSDK.Address, curPath []string, execDepth uint64, f func([]string) (bool, error)) (bool, error) {
func (b *blobovniczas) iterateSorted(addr *oid.Address, curPath []string, execDepth uint64, f func([]string) (bool, error)) (bool, error) {
indices := indexSlice(b.blzShallowWidth)
hrw.SortSliceByValue(indices, addressHash(addr, filepath.Join(curPath...)))
@ -934,11 +934,11 @@ func (b *blobovniczas) openBlobovnicza(p string) (*blobovnicza.Blobovnicza, erro
}
// returns hash of the object address.
func addressHash(addr *addressSDK.Address, path string) uint64 {
func addressHash(addr *oid.Address, path string) uint64 {
var a string
if addr != nil {
a = addr.String()
a = addr.EncodeToString()
}
return hrw.Hash([]byte(a + path))

4
pkg/local_object_storage/blobstor/blobovnicza_test.go
View file

@ -10,7 +10,7 @@ import (
apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status"
cidtest "github.com/nspcc-dev/neofs-sdk-go/container/id/test"
objectSDK "github.com/nspcc-dev/neofs-sdk-go/object"
addressSDK "github.com/nspcc-dev/neofs-sdk-go/object/address"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
oidtest "github.com/nspcc-dev/neofs-sdk-go/object/id/test"
"github.com/stretchr/testify/require"
)
@ -63,7 +63,7 @@ func TestBlobovniczas(t *testing.T) {
objSz := uint64(szLim / 2)
addrList := make([]*addressSDK.Address, 0)
addrList := make([]oid.Address, 0)
minFitObjNum := width * depth * szLim / objSz
for i := uint64(0); i < minFitObjNum; i++ {

10
pkg/local_object_storage/blobstor/exists.go
View file

@ -6,7 +6,7 @@ import (
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree"
addressSDK "github.com/nspcc-dev/neofs-sdk-go/object/address"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"go.uber.org/zap"
)
@ -64,7 +64,7 @@ func (b *BlobStor) Exists(prm *ExistsPrm) (*ExistsRes, error) {
}
// checks if object is presented in shallow dir.
func (b *BlobStor) existsBig(addr *addressSDK.Address) (bool, error) {
func (b *BlobStor) existsBig(addr oid.Address) (bool, error) {
_, err := b.fsTree.Exists(addr)
if errors.Is(err, fstree.ErrFileNotFound) {
return false, nil
@ -74,18 +74,18 @@ func (b *BlobStor) existsBig(addr *addressSDK.Address) (bool, error) {
}
// existsSmall checks if object is presented in blobovnicza.
func (b *BlobStor) existsSmall(addr *addressSDK.Address) (bool, error) {
func (b *BlobStor) existsSmall(addr oid.Address) (bool, error) {
return b.blobovniczas.existsSmall(addr)
}
func (b *blobovniczas) existsSmall(addr *addressSDK.Address) (bool, error) {
func (b *blobovniczas) existsSmall(addr oid.Address) (bool, error) {
activeCache := make(map[string]struct{})
prm := new(blobovnicza.GetPrm)
prm.SetAddress(addr)
var found bool
err := b.iterateSortedLeaves(addr, func(p string) (bool, error) {
err := b.iterateSortedLeaves(&addr, func(p string) (bool, error) {
dirPath := filepath.Dir(p)
_, ok := activeCache[dirPath]

4
pkg/local_object_storage/blobstor/exists_test.go
View file

@ -7,7 +7,7 @@ import (
objectCore "github.com/nspcc-dev/neofs-node/pkg/core/object"
objectSDK "github.com/nspcc-dev/neofs-sdk-go/object"
objecttest "github.com/nspcc-dev/neofs-sdk-go/object/address/test"
oidtest "github.com/nspcc-dev/neofs-sdk-go/object/id/test"
"github.com/stretchr/testify/require"
)
@ -45,7 +45,7 @@ func TestExists(t *testing.T) {
require.True(t, res.Exists())
}
prm.SetAddress(objecttest.Address())
prm.SetAddress(oidtest.Address())
res, err := b.Exists(prm)
require.NoError(t, err)
require.False(t, res.Exists())

46
pkg/local_object_storage/blobstor/fstree/fstree.go
View file

@ -11,8 +11,7 @@ import (
"github.com/nspcc-dev/neofs-node/pkg/util"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
addressSDK "github.com/nspcc-dev/neofs-sdk-go/object/address"
oidSDK "github.com/nspcc-dev/neofs-sdk-go/object/id"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
)
// FSTree represents an object storage as a filesystem tree.
@ -42,44 +41,41 @@ const (
// ErrFileNotFound is returned when file is missing.
var ErrFileNotFound = errors.New("file not found")
func stringifyAddress(addr *addressSDK.Address) string {
id, _ := addr.ObjectID()
cnr, _ := addr.ContainerID()
return id.EncodeToString() + "." + cnr.EncodeToString()
func stringifyAddress(addr oid.Address) string {
return addr.Object().EncodeToString() + "." + addr.Container().EncodeToString()
}
func addressFromString(s string) (*addressSDK.Address, error) {
func addressFromString(s string) (*oid.Address, error) {
ss := strings.SplitN(s, ".", 2)
if len(ss) != 2 {
return nil, errors.New("invalid address")
}
var oid oidSDK.ID
if err := oid.DecodeString(ss[0]); err != nil {
var obj oid.ID
if err := obj.DecodeString(ss[0]); err != nil {
return nil, err
}
var id cid.ID
if err := id.DecodeString(ss[1]); err != nil {
var cnr cid.ID
if err := cnr.DecodeString(ss[1]); err != nil {
return nil, err
}
addr := addressSDK.NewAddress()
addr.SetObjectID(oid)
addr.SetContainerID(id)
var addr oid.Address
addr.SetObject(obj)
addr.SetContainer(cnr)
return addr, nil
return &addr, nil
}
// IterationPrm contains iteraction parameters.
type IterationPrm struct {
handler func(addr *addressSDK.Address, data []byte) error
handler func(addr oid.Address, data []byte) error
ignoreErrors bool
}
// WithHandler sets a function to call on each object.
func (p *IterationPrm) WithHandler(f func(addr *addressSDK.Address, data []byte) error) *IterationPrm {
func (p *IterationPrm) WithHandler(f func(addr oid.Address, data []byte) error) *IterationPrm {
p.handler = f
return p
}
@ -138,7 +134,7 @@ func (t *FSTree) iterate(depth int, curPath []string, prm *IterationPrm) error {
return err
}
if err := prm.handler(addr, data); err != nil {
if err := prm.handler(*addr, data); err != nil {
// Error occurred in handler, outside of our scope, needs to be reported.
return err
}
@ -147,7 +143,7 @@ func (t *FSTree) iterate(depth int, curPath []string, prm *IterationPrm) error {
return nil
}
func (t *FSTree) treePath(addr *addressSDK.Address) string {
func (t *FSTree) treePath(addr oid.Address) string {
sAddr := stringifyAddress(addr)
dirs := make([]string, 0, t.Depth+1+1) // 1 for root, 1 for file
@ -164,7 +160,7 @@ func (t *FSTree) treePath(addr *addressSDK.Address) string {
}
// Delete removes the object with the specified address from the storage.
func (t *FSTree) Delete(addr *addressSDK.Address) error {
func (t *FSTree) Delete(addr oid.Address) error {
p, err := t.Exists(addr)
if err != nil {
return err
@ -175,7 +171,7 @@ func (t *FSTree) Delete(addr *addressSDK.Address) error {
// Exists returns the path to the file with object contents if it exists in the storage
// and an error otherwise.
func (t *FSTree) Exists(addr *addressSDK.Address) (string, error) {
func (t *FSTree) Exists(addr oid.Address) (string, error) {
p := t.treePath(addr)
_, err := os.Stat(p)
@ -187,7 +183,7 @@ func (t *FSTree) Exists(addr *addressSDK.Address) (string, error) {
}
// Put puts an object in the storage.
func (t *FSTree) Put(addr *addressSDK.Address, data []byte) error {
func (t *FSTree) Put(addr oid.Address, data []byte) error {
p := t.treePath(addr)
if err := util.MkdirAllX(filepath.Dir(p), t.Permissions); err != nil {
@ -198,7 +194,7 @@ func (t *FSTree) Put(addr *addressSDK.Address, data []byte) error {
}
// PutStream puts executes handler on a file opened for write.
func (t *FSTree) PutStream(addr *addressSDK.Address, handler func(*os.File) error) error {
func (t *FSTree) PutStream(addr oid.Address, handler func(*os.File) error) error {
p := t.treePath(addr)
if err := util.MkdirAllX(filepath.Dir(p), t.Permissions); err != nil {
@ -215,7 +211,7 @@ func (t *FSTree) PutStream(addr *addressSDK.Address, handler func(*os.File) erro
}
// Get returns an object from the storage by address.
func (t *FSTree) Get(addr *addressSDK.Address) ([]byte, error) {
func (t *FSTree) Get(addr oid.Address) ([]byte, error) {
p := t.treePath(addr)
if _, err := os.Stat(p); os.IsNotExist(err) {

38
pkg/local_object_storage/blobstor/fstree/fstree_test.go
View file

@ -8,17 +8,17 @@ import (
"testing"
"github.com/nspcc-dev/neofs-node/pkg/util"
addressSDK "github.com/nspcc-dev/neofs-sdk-go/object/address"
objecttest "github.com/nspcc-dev/neofs-sdk-go/object/address/test"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
oidtest "github.com/nspcc-dev/neofs-sdk-go/object/id/test"
"github.com/stretchr/testify/require"
)
func TestAddressToString(t *testing.T) {
addr := objecttest.Address()
addr := oidtest.Address()
s := stringifyAddress(addr)
actual, err := addressFromString(s)
require.NoError(t, err)
require.Equal(t, addr, actual)
require.Equal(t, addr, *actual)
}
func TestFSTree(t *testing.T) {
@ -36,28 +36,28 @@ func TestFSTree(t *testing.T) {
}
const count = 3
var addrs []*addressSDK.Address
var addrs []oid.Address
store := map[string][]byte{}
for i := 0; i < count; i++ {
a := objecttest.Address()
a := oidtest.Address()
addrs = append(addrs, a)
data := make([]byte, 10)
_, _ = rand.Read(data[:])
require.NoError(t, fs.Put(a, data))
store[a.String()] = data
store[a.EncodeToString()] = data
}
t.Run("get", func(t *testing.T) {
for _, a := range addrs {
actual, err := fs.Get(a)
require.NoError(t, err)
require.Equal(t, store[a.String()], actual)
require.Equal(t, store[a.EncodeToString()], actual)
}
_, err := fs.Get(objecttest.Address())
_, err := fs.Get(oidtest.Address())
require.Error(t, err)
})
@ -67,16 +67,16 @@ func TestFSTree(t *testing.T) {
require.NoError(t, err)
}
_, err := fs.Exists(objecttest.Address())
_, err := fs.Exists(oidtest.Address())
require.Error(t, err)
})
t.Run("iterate", func(t *testing.T) {
n := 0
err := fs.Iterate(new(IterationPrm).WithHandler(func(addr *addressSDK.Address, data []byte) error {
err := fs.Iterate(new(IterationPrm).WithHandler(func(addr oid.Address, data []byte) error {
n++
expected, ok := store[addr.String()]
require.True(t, ok, "object %s was not found", addr.String())
expected, ok := store[addr.EncodeToString()]
require.True(t, ok, "object %s was not found", addr.EncodeToString())
require.Equal(t, data, expected)
return nil
}))
@ -87,7 +87,7 @@ func TestFSTree(t *testing.T) {
t.Run("leave early", func(t *testing.T) {
n := 0
errStop := errors.New("stop")
err := fs.Iterate(new(IterationPrm).WithHandler(func(addr *addressSDK.Address, data []byte) error {
err := fs.Iterate(new(IterationPrm).WithHandler(func(addr oid.Address, data []byte) error {
if n++; n == count-1 {
return errStop
}
@ -105,16 +105,16 @@ func TestFSTree(t *testing.T) {
require.NoError(t, os.Mkdir(filepath.Join(fs.RootPath, "ZZ"), 0))
// Unreadable file.
p := fs.treePath(objecttest.Address())
p := fs.treePath(oidtest.Address())
require.NoError(t, util.MkdirAllX(filepath.Dir(p), fs.Permissions))
require.NoError(t, os.WriteFile(p, []byte{1, 2, 3}, 0))
// Invalid address.
p = fs.treePath(objecttest.Address()) + ".invalid"
p = fs.treePath(oidtest.Address()) + ".invalid"
require.NoError(t, util.MkdirAllX(filepath.Dir(p), fs.Permissions))
require.NoError(t, os.WriteFile(p, []byte{1, 2, 3}, fs.Permissions))
err := fs.Iterate(new(IterationPrm).WithHandler(func(addr *addressSDK.Address, data []byte) error {
err := fs.Iterate(new(IterationPrm).WithHandler(func(addr oid.Address, data []byte) error {
n++
return nil
}).WithIgnoreErrors(true))
@ -124,7 +124,7 @@ func TestFSTree(t *testing.T) {
t.Run("error from handler is returned", func(t *testing.T) {
expectedErr := errors.New("expected error")
n := 0
err := fs.Iterate(new(IterationPrm).WithHandler(func(addr *addressSDK.Address, data []byte) error {
err := fs.Iterate(new(IterationPrm).WithHandler(func(addr oid.Address, data []byte) error {
n++
if n == count/2 { // process some iterations
return expectedErr
@ -146,6 +146,6 @@ func TestFSTree(t *testing.T) {
_, err = fs.Exists(addrs[1])
require.NoError(t, err)
require.Error(t, fs.Delete(objecttest.Address()))
require.Error(t, fs.Delete(oidtest.Address()))
})
}

4
pkg/local_object_storage/blobstor/iterate.go
View file

@ -6,7 +6,7 @@ import (
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree"
"github.com/nspcc-dev/neofs-sdk-go/object"
addressSDK "github.com/nspcc-dev/neofs-sdk-go/object/address"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
)
// IterationElement represents a unit of elements through which Iterate operation passes.
@ -88,7 +88,7 @@ func (b *BlobStor) Iterate(prm IteratePrm) (*IterateRes, error) {
elem.blzID = nil
err = b.fsTree.Iterate(new(fstree.IterationPrm).WithHandler(func(_ *addressSDK.Address, data []byte) error {
err = b.fsTree.Iterate(new(fstree.IterationPrm).WithHandler(func(_ oid.Address, data []byte) error {
// decompress the data
elem.data, err = b.decompressor(data)
if err != nil {

30
pkg/local_object_storage/blobstor/iterate_test.go
View file

@ -12,8 +12,8 @@ import (
"github.com/klauspost/compress/zstd"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza"
"github.com/nspcc-dev/neofs-sdk-go/object"
addressSDK "github.com/nspcc-dev/neofs-sdk-go/object/address"
objecttest "github.com/nspcc-dev/neofs-sdk-go/object/address/test"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
oidtest "github.com/nspcc-dev/neofs-sdk-go/object/id/test"
"github.com/stretchr/testify/require"
)
@ -45,7 +45,7 @@ func TestIterateObjects(t *testing.T) {
type addrData struct {
big bool
addr *addressSDK.Address
addr oid.Address
data []byte
}
@ -62,7 +62,7 @@ func TestIterateObjects(t *testing.T) {
data := make([]byte, sz)
binary.BigEndian.PutUint64(data, i)
addr := objecttest.Address()
addr := oidtest.Address()
mObjs[string(data)] = addrData{
big: big,
@ -114,15 +114,13 @@ func TestIterate_IgnoreErrors(t *testing.T) {
require.NoError(t, bs.Open())
require.NoError(t, bs.Init())
addrs := make([]*addressSDK.Address, objCount)
addrs := make([]oid.Address, objCount)
for i := range addrs {
addrs[i] = objecttest.Address()
id, _ := addrs[i].ObjectID()
cnr, _ := addrs[i].ContainerID()
addrs[i] = oidtest.Address()
obj := object.New()
obj.SetContainerID(cnr)
obj.SetID(id)
obj.SetContainerID(addrs[i].Container())
obj.SetID(addrs[i].Object())
obj.SetPayload(make([]byte, smallSize<<(i%2)))
objData, err := obj.Marshal()
@ -142,9 +140,9 @@ func TestIterate_IgnoreErrors(t *testing.T) {
rawData[i] ^= 0xFF
}
// Will be put uncompressed but fetched as compressed because of magic.
_, err = bs.PutRaw(objecttest.Address(), rawData, false)
_, err = bs.PutRaw(oidtest.Address(), rawData, false)
require.NoError(t, err)
require.NoError(t, bs.fsTree.Put(objecttest.Address(), rawData))
require.NoError(t, bs.fsTree.Put(oidtest.Address(), rawData))
require.NoError(t, bs.Close())
@ -178,7 +176,7 @@ func TestIterate_IgnoreErrors(t *testing.T) {
prm.IgnoreErrors()
t.Run("skip invalid objects", func(t *testing.T) {
actual := make([]*addressSDK.Address, 0, len(addrs))
actual := make([]oid.Address, 0, len(addrs))
prm.SetIterationHandler(func(e IterationElement) error {
obj := object.New()
err := obj.Unmarshal(e.data)
@ -186,11 +184,11 @@ func TestIterate_IgnoreErrors(t *testing.T) {
return err
}
addr := addressSDK.NewAddress()
var addr oid.Address
cnr, _ := obj.ContainerID()
addr.SetContainerID(cnr)
addr.SetContainer(cnr)
id, _ := obj.ID()
addr.SetObjectID(id)
addr.SetObject(id)
actual = append(actual, addr)
return nil
})

4
pkg/local_object_storage/blobstor/put.go
View file

@ -9,7 +9,7 @@ import (
"github.com/nspcc-dev/neofs-node/pkg/core/object"
storagelog "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/internal/log"
objectSDK "github.com/nspcc-dev/neofs-sdk-go/object"
addressSDK "github.com/nspcc-dev/neofs-sdk-go/object/address"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
)
// PutPrm groups the parameters of Put operation.
@ -72,7 +72,7 @@ func (b *BlobStor) NeedsCompression(obj *objectSDK.Object) bool {
}
// PutRaw saves an already marshaled object in BLOB storage.
func (b *BlobStor) PutRaw(addr *addressSDK.Address, data []byte, compress bool) (*PutRes, error) {
func (b *BlobStor) PutRaw(addr oid.Address, data []byte, compress bool) (*PutRes, error) {
big := b.isBig(data)
if big {

6
pkg/local_object_storage/blobstor/util.go
View file

@ -3,15 +3,15 @@ package blobstor
import (
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza"
"github.com/nspcc-dev/neofs-sdk-go/object"
addressSDK "github.com/nspcc-dev/neofs-sdk-go/object/address"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
)
type address struct {
addr *addressSDK.Address
addr oid.Address
}
// SetAddress sets the address of the requested object.
func (a *address) SetAddress(addr *addressSDK.Address) {
func (a *address) SetAddress(addr oid.Address) {
a.addr = addr
}