[#1523] blobstor: Unify parameters for Put operation

Signed-off-by: Evgenii Stratonikov <evgeniy@nspcc.ru>
This commit is contained in:
Evgenii Stratonikov 2022-07-06 16:41:35 +03:00 committed by fyrchik
parent 7d6df543d7
commit 9eb018672c
13 changed files with 91 additions and 73 deletions

View file

@ -142,10 +142,10 @@ func indexSlice(number uint64) []uint64 {
// save object in the maximum weight blobobnicza. // save object in the maximum weight blobobnicza.
// //
// returns error if could not save object in any blobovnicza. // returns error if could not save object in any blobovnicza.
func (b *Blobovniczas) Put(addr oid.Address, data []byte) (*blobovnicza.ID, error) { func (b *Blobovniczas) Put(prm common.PutPrm) (common.PutRes, error) {
var prm blobovnicza.PutPrm var putPrm blobovnicza.PutPrm
prm.SetAddress(addr) putPrm.SetAddress(prm.Address)
prm.SetMarshaledObject(data) putPrm.SetMarshaledObject(prm.RawData)
var ( var (
fn func(string) (bool, error) fn func(string) (bool, error)
@ -162,7 +162,7 @@ func (b *Blobovniczas) Put(addr oid.Address, data []byte) (*blobovnicza.ID, erro
return false, nil return false, nil
} }
if _, err := active.blz.Put(prm); err != nil { if _, err := active.blz.Put(putPrm); err != nil {
// check if blobovnicza is full // check if blobovnicza is full
if errors.Is(err, blobovnicza.ErrFull) { if errors.Is(err, blobovnicza.ErrFull) {
b.log.Debug("blobovnicza overflowed", b.log.Debug("blobovnicza overflowed",
@ -193,18 +193,18 @@ func (b *Blobovniczas) Put(addr oid.Address, data []byte) (*blobovnicza.ID, erro
id = blobovnicza.NewIDFromBytes([]byte(p)) id = blobovnicza.NewIDFromBytes([]byte(p))
storagelog.Write(b.log, storagelog.AddressField(addr), storagelog.OpField("Blobovniczas PUT")) storagelog.Write(b.log, storagelog.AddressField(prm.Address), storagelog.OpField("Blobovniczas PUT"))
return true, nil return true, nil
} }
if err := b.iterateDeepest(addr, fn); err != nil { if err := b.iterateDeepest(prm.Address, fn); err != nil {
return nil, err return common.PutRes{}, err
} else if id == nil { } else if id == nil {
return nil, errPutFailed return common.PutRes{}, errPutFailed
} }
return id, nil return common.PutRes{BlobovniczaID: id}, nil
} }
// Get reads object from blobovnicza tree. // Get reads object from blobovnicza tree.

View file

@ -73,12 +73,12 @@ func TestBlobovniczas(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
// save object in blobovnicza // save object in blobovnicza
id, err := b.Put(addr, d) pRes, err := b.Put(common.PutPrm{Address: addr, RawData: d})
require.NoError(t, err, i) require.NoError(t, err, i)
// get w/ blobovnicza ID // get w/ blobovnicza ID
var prm common.GetPrm var prm common.GetPrm
prm.BlobovniczaID = id prm.BlobovniczaID = pRes.BlobovniczaID
prm.Address = addr prm.Address = addr
res, err := b.Get(prm) res, err := b.Get(prm)
@ -94,7 +94,7 @@ func TestBlobovniczas(t *testing.T) {
// get range w/ blobovnicza ID // get range w/ blobovnicza ID
var rngPrm common.GetRangePrm var rngPrm common.GetRangePrm
rngPrm.BlobovniczaID = id rngPrm.BlobovniczaID = pRes.BlobovniczaID
rngPrm.Address = addr rngPrm.Address = addr
payload := obj.Payload() payload := obj.Payload()

View file

@ -48,13 +48,13 @@ func TestCompression(t *testing.T) {
} }
testPut := func(t *testing.T, b *BlobStor, i int) { testPut := func(t *testing.T, b *BlobStor, i int) {
var prm PutPrm var prm common.PutPrm
prm.SetObject(smallObj[i]) prm.Object = smallObj[i]
_, err = b.Put(prm) _, err = b.Put(prm)
require.NoError(t, err) require.NoError(t, err)
prm = PutPrm{} prm = common.PutPrm{}
prm.SetObject(bigObj[i]) prm.Object = bigObj[i]
_, err = b.Put(prm) _, err = b.Put(prm)
require.NoError(t, err) require.NoError(t, err)
} }

View file

@ -0,0 +1,19 @@
package common
import (
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza"
objectSDK "github.com/nspcc-dev/neofs-sdk-go/object"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
)
// PutPrm groups the parameters of Put operation.
type PutPrm struct {
Address oid.Address
Object *objectSDK.Object
RawData []byte
}
// PutRes groups the resulting values of Put operation.
type PutRes struct {
BlobovniczaID *blobovnicza.ID
}

View file

@ -32,8 +32,8 @@ func TestExists(t *testing.T) {
} }
for i := range objects { for i := range objects {
var prm PutPrm var prm common.PutPrm
prm.SetObject(objects[i]) prm.Object = objects[i]
_, err = b.Put(prm) _, err = b.Put(prm)
require.NoError(t, err) require.NoError(t, err)
} }

View file

@ -206,14 +206,14 @@ func (t *FSTree) Exists(addr oid.Address) (string, error) {
} }
// Put puts an object in the storage. // Put puts an object in the storage.
func (t *FSTree) Put(addr oid.Address, data []byte) error { func (t *FSTree) Put(prm common.PutPrm) (common.PutRes, error) {
p := t.treePath(addr) p := t.treePath(prm.Address)
if err := util.MkdirAllX(filepath.Dir(p), t.Permissions); err != nil { if err := util.MkdirAllX(filepath.Dir(p), t.Permissions); err != nil {
return err return common.PutRes{}, err
} }
return os.WriteFile(p, data, t.Permissions) return common.PutRes{}, os.WriteFile(p, prm.RawData, t.Permissions)
} }
// PutStream puts executes handler on a file opened for write. // PutStream puts executes handler on a file opened for write.

View file

@ -47,7 +47,9 @@ func TestFSTree(t *testing.T) {
data := make([]byte, 10) data := make([]byte, 10)
_, _ = rand.Read(data[:]) _, _ = rand.Read(data[:])
require.NoError(t, fs.Put(a, data))
_, err := fs.Put(common.PutPrm{Address: a, RawData: data})
require.NoError(t, err)
store[a.EncodeToString()] = data store[a.EncodeToString()] = data
} }

View file

@ -6,6 +6,7 @@ import (
"testing" "testing"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/common"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id" oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
oidtest "github.com/nspcc-dev/neofs-sdk-go/object/id/test" oidtest "github.com/nspcc-dev/neofs-sdk-go/object/id/test"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
@ -66,7 +67,7 @@ func TestIterateObjects(t *testing.T) {
} }
for _, v := range mObjs { for _, v := range mObjs {
_, err := blobStor.PutRaw(v.addr, v.data, true) _, err := blobStor.PutRaw(common.PutPrm{Address: v.addr, RawData: v.data}, true)
require.NoError(t, err) require.NoError(t, err)
} }

View file

@ -6,21 +6,11 @@ import (
"github.com/klauspost/compress/zstd" "github.com/klauspost/compress/zstd"
"github.com/nspcc-dev/neofs-node/pkg/core/object" "github.com/nspcc-dev/neofs-node/pkg/core/object"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/common"
storagelog "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/internal/log" storagelog "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/internal/log"
objectSDK "github.com/nspcc-dev/neofs-sdk-go/object" objectSDK "github.com/nspcc-dev/neofs-sdk-go/object"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
) )
// PutPrm groups the parameters of Put operation.
type PutPrm struct {
rwObject
}
// PutRes groups the resulting values of Put operation.
type PutRes struct {
roBlobovniczaID
}
// Put saves the object in BLOB storage. // Put saves the object in BLOB storage.
// //
// If object is "big", BlobStor saves the object in shallow dir. // If object is "big", BlobStor saves the object in shallow dir.
@ -29,14 +19,18 @@ type PutRes struct {
// //
// Returns any error encountered that // Returns any error encountered that
// did not allow to completely save the object. // did not allow to completely save the object.
func (b *BlobStor) Put(prm PutPrm) (PutRes, error) { func (b *BlobStor) Put(prm common.PutPrm) (common.PutRes, error) {
prm.Address = object.AddressOf(prm.Object)
if prm.RawData == nil {
// marshal object // marshal object
data, err := prm.obj.Marshal() data, err := prm.Object.Marshal()
if err != nil { if err != nil {
return PutRes{}, fmt.Errorf("could not marshal the object: %w", err) return common.PutRes{}, fmt.Errorf("could not marshal the object: %w", err)
}
prm.RawData = data
} }
return b.PutRaw(object.AddressOf(prm.obj), data, b.NeedsCompression(prm.obj)) return b.PutRaw(prm, b.NeedsCompression(prm.Object))
} }
// NeedsCompression returns true if the object should be compressed. // NeedsCompression returns true if the object should be compressed.
@ -48,46 +42,37 @@ func (b *BlobStor) NeedsCompression(obj *objectSDK.Object) bool {
} }
// PutRaw saves an already marshaled object in BLOB storage. // PutRaw saves an already marshaled object in BLOB storage.
func (b *BlobStor) PutRaw(addr oid.Address, data []byte, compress bool) (PutRes, error) { func (b *BlobStor) PutRaw(prm common.PutPrm, compress bool) (common.PutRes, error) {
big := b.isBig(data) big := b.isBig(prm.RawData)
if big { if big {
var err error var err error
if compress { if compress {
err = b.fsTree.PutStream(addr, func(f *os.File) error { err = b.fsTree.PutStream(prm.Address, func(f *os.File) error {
enc, _ := zstd.NewWriter(f) // nil error if no options are provided enc, _ := zstd.NewWriter(f) // nil error if no options are provided
if _, err := enc.Write(data); err != nil { if _, err := enc.Write(prm.RawData); err != nil {
return err return err
} }
return enc.Close() return enc.Close()
}) })
} else { } else {
err = b.fsTree.Put(addr, data) _, err = b.fsTree.Put(prm)
} }
if err != nil { if err != nil {
return PutRes{}, err return common.PutRes{}, err
} }
storagelog.Write(b.log, storagelog.AddressField(addr), storagelog.OpField("fstree PUT")) storagelog.Write(b.log, storagelog.AddressField(prm.Address), storagelog.OpField("fstree PUT"))
return PutRes{}, nil return common.PutRes{}, nil
} }
if compress { if compress {
data = b.CConfig.Compress(data) prm.RawData = b.CConfig.Compress(prm.RawData)
} }
// save object in blobovnicza // save object in blobovnicza
res, err := b.blobovniczas.Put(addr, data) return b.blobovniczas.Put(prm)
if err != nil {
return PutRes{}, err
}
return PutRes{
roBlobovniczaID: roBlobovniczaID{
blobovniczaID: res,
},
}, nil
} }
// checks if object is "big". // checks if object is "big".

View file

@ -8,6 +8,7 @@ import (
"github.com/nspcc-dev/neofs-node/pkg/core/object" "github.com/nspcc-dev/neofs-node/pkg/core/object"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/common"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree"
meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase" meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/pilorama" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/pilorama"
@ -109,7 +110,8 @@ func TestRefillMetabaseCorrupted(t *testing.T) {
Depth: 1, Depth: 1,
Info: sh.blobStor.DumpInfo(), Info: sh.blobStor.DumpInfo(),
} }
require.NoError(t, fs.Put(addr, []byte("not an object"))) _, err = fs.Put(common.PutPrm{Address: addr, RawData: []byte("not an object")})
require.NoError(t, err)
sh = New( sh = New(
WithBlobStorOptions(blobOpts...), WithBlobStorOptions(blobOpts...),

View file

@ -3,7 +3,7 @@ package shard
import ( import (
"fmt" "fmt"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/common"
meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase" meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
"github.com/nspcc-dev/neofs-sdk-go/object" "github.com/nspcc-dev/neofs-sdk-go/object"
"go.uber.org/zap" "go.uber.org/zap"
@ -34,8 +34,8 @@ func (s *Shard) Put(prm PutPrm) (PutRes, error) {
return PutRes{}, ErrReadOnlyMode return PutRes{}, ErrReadOnlyMode
} }
var putPrm blobstor.PutPrm // form Put parameters var putPrm common.PutPrm // form Put parameters
putPrm.SetObject(prm.obj) putPrm.Object = prm.obj
// exist check are not performed there, these checks should be executed // exist check are not performed there, these checks should be executed
// ahead of `Put` by storage engine // ahead of `Put` by storage engine
@ -51,7 +51,7 @@ func (s *Shard) Put(prm PutPrm) (PutRes, error) {
var ( var (
err error err error
res blobstor.PutRes res common.PutRes
) )
if res, err = s.blobStor.Put(putPrm); err != nil { if res, err = s.blobStor.Put(putPrm); err != nil {
@ -61,7 +61,7 @@ func (s *Shard) Put(prm PutPrm) (PutRes, error) {
if !m.NoMetabase() { if !m.NoMetabase() {
var pPrm meta.PutPrm var pPrm meta.PutPrm
pPrm.SetObject(prm.obj) pPrm.SetObject(prm.obj)
pPrm.SetBlobovniczaID(res.BlobovniczaID()) pPrm.SetBlobovniczaID(res.BlobovniczaID)
if _, err := s.metaBase.Put(pPrm); err != nil { if _, err := s.metaBase.Put(pPrm); err != nil {
// may we need to handle this case in a special way // may we need to handle this case in a special way
// since the object has been successfully written to BlobStor // since the object has been successfully written to BlobStor

View file

@ -6,7 +6,7 @@ import (
"github.com/mr-tron/base58" "github.com/mr-tron/base58"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobovnicza"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/common"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree"
meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase" meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
"github.com/nspcc-dev/neofs-sdk-go/object" "github.com/nspcc-dev/neofs-sdk-go/object"
@ -154,7 +154,11 @@ func (c *cache) flushBigObjects() {
_, compress := c.compressFlags[sAddr] _, compress := c.compressFlags[sAddr]
c.mtx.Unlock() c.mtx.Unlock()
if _, err := c.blobstor.PutRaw(addr, data, compress); err != nil { var prm common.PutPrm
prm.Address = addr
prm.RawData = data
if _, err := c.blobstor.PutRaw(common.PutPrm{Address: addr, RawData: data}, compress); err != nil {
c.log.Error("cant flush object to blobstor", zap.Error(err)) c.log.Error("cant flush object to blobstor", zap.Error(err))
return nil return nil
} }
@ -227,15 +231,15 @@ func (c *cache) writeObject(obj *object.Object, metaOnly bool) error {
var id *blobovnicza.ID var id *blobovnicza.ID
if !metaOnly { if !metaOnly {
var prm blobstor.PutPrm var prm common.PutPrm
prm.SetObject(obj) prm.Object = obj
res, err := c.blobstor.Put(prm) res, err := c.blobstor.Put(prm)
if err != nil { if err != nil {
return err return err
} }
id = res.BlobovniczaID() id = res.BlobovniczaID
} }
var pPrm meta.PutPrm var pPrm meta.PutPrm

View file

@ -5,6 +5,7 @@ import (
"time" "time"
"github.com/nspcc-dev/neofs-node/pkg/core/object" "github.com/nspcc-dev/neofs-node/pkg/core/object"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/common"
storagelog "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/internal/log" storagelog "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/internal/log"
"go.etcd.io/bbolt" "go.etcd.io/bbolt"
"go.uber.org/zap" "go.uber.org/zap"
@ -113,7 +114,11 @@ func (c *cache) persistBigObject(objInfo objectInfo) {
cacheSz := c.estimateCacheSize() cacheSz := c.estimateCacheSize()
metaIndex := 0 metaIndex := 0
if c.incSizeFS(cacheSz) <= c.maxCacheSize { if c.incSizeFS(cacheSz) <= c.maxCacheSize {
err := c.fsTree.Put(object.AddressOf(objInfo.obj), objInfo.data) var prm common.PutPrm
prm.Address = object.AddressOf(objInfo.obj)
prm.RawData = objInfo.data
_, err := c.fsTree.Put(prm)
if err == nil { if err == nil {
metaIndex = 1 metaIndex = 1
if c.blobstor.NeedsCompression(objInfo.obj) { if c.blobstor.NeedsCompression(objInfo.obj) {