[#1418] blobstor: Do not use pointers as parameters

Signed-off-by: Pavel Karpy <carpawell@nspcc.ru>
experimental
Pavel Karpy 2022-05-23 17:15:16 +03:00 committed by fyrchik
parent babd382ba5
commit 281befec67
26 changed files with 89 additions and 66 deletions

View File

@ -85,7 +85,7 @@ func objectInspectCmd(cmd *cobra.Command, _ []string) {
defer blz.Close()
prm := new(blobovnicza.GetPrm)
var prm blobovnicza.GetPrm
prm.SetAddress(addr)
res, err := blz.Get(prm)
common.ExitOnErr(cmd, common.Errf("could not fetch object: %w", err))

View File

@ -36,7 +36,7 @@ func testPutGet(t *testing.T, blz *Blobovnicza, addr oid.Address, sz uint64, ass
}
func testGet(t *testing.T, blz *Blobovnicza, addr oid.Address, expObj []byte, assertErr func(error) bool) {
pGet := new(GetPrm)
var pGet GetPrm
pGet.SetAddress(addr)
// try to read object from Blobovnicza
@ -85,7 +85,7 @@ func TestBlobovnicza(t *testing.T) {
addr := testPutGet(t, blz, oidtest.Address(), filled, nil, nil)
// remove the object
dPrm := new(DeletePrm)
var dPrm DeletePrm
dPrm.SetAddress(addr)
_, err := blz.Delete(dPrm)

View File

@ -29,7 +29,7 @@ func (p *DeletePrm) SetAddress(addr oid.Address) {
// Returns an error of type apistatus.ObjectNotFound if the object to be deleted is not in blobovnicza.
//
// Should not be called in read-only configuration.
func (b *Blobovnicza) Delete(prm *DeletePrm) (*DeleteRes, error) {
func (b *Blobovnicza) Delete(prm DeletePrm) (*DeleteRes, error) {
addrKey := addressKey(prm.addr)
removed := false

View File

@ -24,7 +24,7 @@ func (p *GetPrm) SetAddress(addr oid.Address) {
}
// Object returns binary representation of the requested object.
func (p *GetRes) Object() []byte {
func (p GetRes) Object() []byte {
return p.obj
}
@ -35,7 +35,7 @@ func (p *GetRes) Object() []byte {
//
// Returns an error of type apistatus.ObjectNotFound if the requested object is not
// presented in Blobovnicza.
func (b *Blobovnicza) Get(prm *GetPrm) (*GetRes, error) {
func (b *Blobovnicza) Get(prm GetPrm) (*GetRes, error) {
var (
data []byte
addrKey = addressKey(prm.addr)

View File

@ -205,8 +205,8 @@ func (b *blobovniczas) put(addr oid.Address, data []byte) (*blobovnicza.ID, erro
//
// If blobocvnicza ID is specified, only this blobovnicza is processed.
// Otherwise, all blobovniczas are processed descending weight.
func (b *blobovniczas) get(prm *GetSmallPrm) (res *GetSmallRes, err error) {
bPrm := new(blobovnicza.GetPrm)
func (b *blobovniczas) get(prm GetSmallPrm) (res *GetSmallRes, err error) {
var bPrm blobovnicza.GetPrm
bPrm.SetAddress(prm.addr)
if prm.blobovniczaID != nil {
@ -255,8 +255,8 @@ func (b *blobovniczas) get(prm *GetSmallPrm) (res *GetSmallRes, err error) {
//
// If blobocvnicza ID is specified, only this blobovnicza is processed.
// Otherwise, all blobovniczas are processed descending weight.
func (b *blobovniczas) delete(prm *DeleteSmallPrm) (res *DeleteSmallRes, err error) {
bPrm := new(blobovnicza.DeletePrm)
func (b *blobovniczas) delete(prm DeleteSmallPrm) (res *DeleteSmallRes, err error) {
var bPrm blobovnicza.DeletePrm
bPrm.SetAddress(prm.addr)
if prm.blobovniczaID != nil {
@ -310,7 +310,7 @@ func (b *blobovniczas) delete(prm *DeleteSmallPrm) (res *DeleteSmallRes, err err
//
// If blobocvnicza ID is specified, only this blobovnicza is processed.
// Otherwise, all blobovniczas are processed descending weight.
func (b *blobovniczas) getRange(prm *GetRangeSmallPrm) (res *GetRangeSmallRes, err error) {
func (b *blobovniczas) getRange(prm GetRangeSmallPrm) (res *GetRangeSmallRes, err error) {
if prm.blobovniczaID != nil {
blz, err := b.openBlobovnicza(prm.blobovniczaID.String())
if err != nil {
@ -360,7 +360,7 @@ func (b *blobovniczas) getRange(prm *GetRangeSmallPrm) (res *GetRangeSmallRes, e
// tries to delete object from particular blobovnicza.
//
// returns no error if object was removed from some blobovnicza of the same level.
func (b *blobovniczas) deleteObjectFromLevel(prm *blobovnicza.DeletePrm, blzPath string, tryActive bool, dp *DeleteSmallPrm) (*DeleteSmallRes, error) {
func (b *blobovniczas) deleteObjectFromLevel(prm blobovnicza.DeletePrm, blzPath string, tryActive bool, dp DeleteSmallPrm) (*DeleteSmallRes, error) {
lvlPath := filepath.Dir(blzPath)
log := b.log.With(
@ -423,7 +423,7 @@ func (b *blobovniczas) deleteObjectFromLevel(prm *blobovnicza.DeletePrm, blzPath
// tries to read object from particular blobovnicza.
//
// returns error if object could not be read from any blobovnicza of the same level.
func (b *blobovniczas) getObjectFromLevel(prm *blobovnicza.GetPrm, blzPath string, tryActive bool) (*GetSmallRes, error) {
func (b *blobovniczas) getObjectFromLevel(prm blobovnicza.GetPrm, blzPath string, tryActive bool) (*GetSmallRes, error) {
lvlPath := filepath.Dir(blzPath)
log := b.log.With(
@ -487,7 +487,7 @@ func (b *blobovniczas) getObjectFromLevel(prm *blobovnicza.GetPrm, blzPath strin
// tries to read range of object payload data from particular blobovnicza.
//
// returns error if object could not be read from any blobovnicza of the same level.
func (b *blobovniczas) getRangeFromLevel(prm *GetRangeSmallPrm, blzPath string, tryActive bool) (*GetRangeSmallRes, error) {
func (b *blobovniczas) getRangeFromLevel(prm GetRangeSmallPrm, blzPath string, tryActive bool) (*GetRangeSmallRes, error) {
lvlPath := filepath.Dir(blzPath)
log := b.log.With(
@ -560,7 +560,7 @@ func (b *blobovniczas) getRangeFromLevel(prm *GetRangeSmallPrm, blzPath string,
}
// removes object from blobovnicza and returns DeleteSmallRes.
func (b *blobovniczas) deleteObject(blz *blobovnicza.Blobovnicza, prm *blobovnicza.DeletePrm, dp *DeleteSmallPrm) (*DeleteSmallRes, error) {
func (b *blobovniczas) deleteObject(blz *blobovnicza.Blobovnicza, prm blobovnicza.DeletePrm, dp DeleteSmallPrm) (*DeleteSmallRes, error) {
_, err := blz.Delete(prm)
if err != nil {
return nil, err
@ -576,7 +576,7 @@ func (b *blobovniczas) deleteObject(blz *blobovnicza.Blobovnicza, prm *blobovnic
}
// reads object from blobovnicza and returns GetSmallRes.
func (b *blobovniczas) getObject(blz *blobovnicza.Blobovnicza, prm *blobovnicza.GetPrm) (*GetSmallRes, error) {
func (b *blobovniczas) getObject(blz *blobovnicza.Blobovnicza, prm blobovnicza.GetPrm) (*GetSmallRes, error) {
res, err := blz.Get(prm)
if err != nil {
return nil, err
@ -602,8 +602,8 @@ func (b *blobovniczas) getObject(blz *blobovnicza.Blobovnicza, prm *blobovnicza.
}
// reads range of object payload data from blobovnicza and returns GetRangeSmallRes.
func (b *blobovniczas) getObjectRange(blz *blobovnicza.Blobovnicza, prm *GetRangeSmallPrm) (*GetRangeSmallRes, error) {
gPrm := new(blobovnicza.GetPrm)
func (b *blobovniczas) getObjectRange(blz *blobovnicza.Blobovnicza, prm GetRangeSmallPrm) (*GetRangeSmallRes, error) {
var gPrm blobovnicza.GetPrm
gPrm.SetAddress(prm.addr)
// we don't use GetRange call for now since blobovnicza

View File

@ -80,7 +80,7 @@ func TestBlobovniczas(t *testing.T) {
require.NoError(t, err)
// get w/ blobovnicza ID
prm := new(GetSmallPrm)
var prm GetSmallPrm
prm.SetBlobovniczaID(id)
prm.SetAddress(addr)
@ -96,7 +96,7 @@ func TestBlobovniczas(t *testing.T) {
require.Equal(t, obj, res.Object())
// get range w/ blobovnicza ID
rngPrm := new(GetRangeSmallPrm)
var rngPrm GetRangeSmallPrm
rngPrm.SetBlobovniczaID(id)
rngPrm.SetAddress(addr)
@ -122,8 +122,8 @@ func TestBlobovniczas(t *testing.T) {
require.Equal(t, payload[off:off+ln], rngRes.RangeData())
}
dPrm := new(DeleteSmallPrm)
gPrm := new(GetSmallPrm)
var dPrm DeleteSmallPrm
var gPrm GetSmallPrm
for i := range addrList {
dPrm.SetAddress(addrList[i])

View File

@ -37,22 +37,22 @@ func TestCompression(t *testing.T) {
}
testGet := func(t *testing.T, b *BlobStor, i int) {
res1, err := b.GetSmall(&GetSmallPrm{address: address{object.AddressOf(smallObj[i])}})
res1, err := b.GetSmall(GetSmallPrm{address: address{object.AddressOf(smallObj[i])}})
require.NoError(t, err)
require.Equal(t, smallObj[i], res1.Object())
res2, err := b.GetBig(&GetBigPrm{address: address{object.AddressOf(bigObj[i])}})
res2, err := b.GetBig(GetBigPrm{address: address{object.AddressOf(bigObj[i])}})
require.NoError(t, err)
require.Equal(t, bigObj[i], res2.Object())
}
testPut := func(t *testing.T, b *BlobStor, i int) {
prm := new(PutPrm)
var prm PutPrm
prm.SetObject(smallObj[i])
_, err = b.Put(prm)
require.NoError(t, err)
prm = new(PutPrm)
prm = PutPrm{}
prm.SetObject(bigObj[i])
_, err = b.Put(prm)
require.NoError(t, err)

View File

@ -22,7 +22,7 @@ type DeleteBigRes struct{}
// to completely remove the object.
//
// Returns an error of type apistatus.ObjectNotFound if there is no object to delete.
func (b *BlobStor) DeleteBig(prm *DeleteBigPrm) (*DeleteBigRes, error) {
func (b *BlobStor) DeleteBig(prm DeleteBigPrm) (*DeleteBigRes, error) {
err := b.fsTree.Delete(prm.addr)
if errors.Is(err, fstree.ErrFileNotFound) {
var errNotFound apistatus.ObjectNotFound

View File

@ -18,6 +18,6 @@ type DeleteSmallRes struct{}
// to completely remove the object.
//
// Returns an error of type apistatus.ObjectNotFound if there is no object to delete.
func (b *BlobStor) DeleteSmall(prm *DeleteSmallPrm) (*DeleteSmallRes, error) {
func (b *BlobStor) DeleteSmall(prm DeleteSmallPrm) (*DeleteSmallRes, error) {
return b.blobovniczas.delete(prm)
}

View File

@ -29,7 +29,7 @@ func (r ExistsRes) Exists() bool {
//
// Returns any error encountered that did not allow
// to completely check object existence.
func (b *BlobStor) Exists(prm *ExistsPrm) (*ExistsRes, error) {
func (b *BlobStor) Exists(prm ExistsPrm) (*ExistsRes, error) {
// check presence in shallow dir first (cheaper)
exists, err := b.existsBig(prm.addr)
@ -81,7 +81,7 @@ func (b *BlobStor) existsSmall(addr oid.Address) (bool, error) {
func (b *blobovniczas) existsSmall(addr oid.Address) (bool, error) {
activeCache := make(map[string]struct{})
prm := new(blobovnicza.GetPrm)
var prm blobovnicza.GetPrm
prm.SetAddress(addr)
var found bool

View File

@ -30,13 +30,13 @@ func TestExists(t *testing.T) {
}
for i := range objects {
prm := new(PutPrm)
var prm PutPrm
prm.SetObject(objects[i])
_, err = b.Put(prm)
require.NoError(t, err)
}
prm := new(ExistsPrm)
var prm ExistsPrm
for i := range objects {
prm.SetAddress(objectCore.AddressOf(objects[i]))

View File

@ -75,23 +75,21 @@ type IterationPrm struct {
}
// WithHandler sets a function to call on each object.
func (p *IterationPrm) WithHandler(f func(addr oid.Address, data []byte) error) *IterationPrm {
func (p *IterationPrm) WithHandler(f func(addr oid.Address, data []byte) error) {
p.handler = f
return p
}
// WithIgnoreErrors sets a flag indicating whether errors should be ignored.
func (p *IterationPrm) WithIgnoreErrors(ignore bool) *IterationPrm {
func (p *IterationPrm) WithIgnoreErrors(ignore bool) {
p.ignoreErrors = ignore
return p
}
// Iterate iterates over all stored objects.
func (t *FSTree) Iterate(prm *IterationPrm) error {
func (t *FSTree) Iterate(prm IterationPrm) error {
return t.iterate(0, []string{t.RootPath}, prm)
}
func (t *FSTree) iterate(depth int, curPath []string, prm *IterationPrm) error {
func (t *FSTree) iterate(depth int, curPath []string, prm IterationPrm) error {
curName := strings.Join(curPath[1:], "")
des, err := os.ReadDir(filepath.Join(curPath...))
if err != nil {

View File

@ -73,13 +73,16 @@ func TestFSTree(t *testing.T) {
t.Run("iterate", func(t *testing.T) {
n := 0
err := fs.Iterate(new(IterationPrm).WithHandler(func(addr oid.Address, data []byte) error {
var iterationPrm IterationPrm
iterationPrm.WithHandler(func(addr oid.Address, data []byte) error {
n++
expected, ok := store[addr.EncodeToString()]
require.True(t, ok, "object %s was not found", addr.EncodeToString())
require.Equal(t, data, expected)
return nil
}))
})
err := fs.Iterate(iterationPrm)
require.NoError(t, err)
require.Equal(t, count, n)
@ -87,12 +90,15 @@ func TestFSTree(t *testing.T) {
t.Run("leave early", func(t *testing.T) {
n := 0
errStop := errors.New("stop")
err := fs.Iterate(new(IterationPrm).WithHandler(func(addr oid.Address, data []byte) error {
iterationPrm.WithHandler(func(addr oid.Address, data []byte) error {
if n++; n == count-1 {
return errStop
}
return nil
}))
})
err := fs.Iterate(iterationPrm)
require.ErrorIs(t, err, errStop)
require.Equal(t, count-1, n)
@ -114,23 +120,29 @@ func TestFSTree(t *testing.T) {
require.NoError(t, util.MkdirAllX(filepath.Dir(p), fs.Permissions))
require.NoError(t, os.WriteFile(p, []byte{1, 2, 3}, fs.Permissions))
err := fs.Iterate(new(IterationPrm).WithHandler(func(addr oid.Address, data []byte) error {
iterationPrm.WithIgnoreErrors(true)
iterationPrm.WithHandler(func(addr oid.Address, data []byte) error {
n++
return nil
}).WithIgnoreErrors(true))
})
err := fs.Iterate(iterationPrm)
require.NoError(t, err)
require.Equal(t, count, n)
t.Run("error from handler is returned", func(t *testing.T) {
expectedErr := errors.New("expected error")
n := 0
err := fs.Iterate(new(IterationPrm).WithHandler(func(addr oid.Address, data []byte) error {
iterationPrm.WithHandler(func(addr oid.Address, data []byte) error {
n++
if n == count/2 { // process some iterations
return expectedErr
}
return nil
}).WithIgnoreErrors(true))
})
err := fs.Iterate(iterationPrm)
require.ErrorIs(t, err, expectedErr)
require.Equal(t, count/2, n)
})

View File

@ -26,7 +26,7 @@ type GetBigRes struct {
//
// Returns an error of type apistatus.ObjectNotFound if the requested object is not
// presented in shallow dir.
func (b *BlobStor) GetBig(prm *GetBigPrm) (*GetBigRes, error) {
func (b *BlobStor) GetBig(prm GetBigPrm) (*GetBigRes, error) {
// get compressed object data
data, err := b.fsTree.Get(prm.addr)
if err != nil {

View File

@ -28,7 +28,7 @@ type GetRangeBigRes struct {
//
// Returns ErrRangeOutOfBounds if the requested object range is out of bounds.
// Returns an error of type apistatus.ObjectNotFound if object is missing.
func (b *BlobStor) GetRangeBig(prm *GetRangeBigPrm) (*GetRangeBigRes, error) {
func (b *BlobStor) GetRangeBig(prm GetRangeBigPrm) (*GetRangeBigRes, error) {
// get compressed object data
data, err := b.fsTree.Get(prm.addr)
if err != nil {

View File

@ -22,6 +22,6 @@ type GetRangeSmallRes struct {
//
// Returns ErrRangeOutOfBounds if the requested object range is out of bounds.
// Returns an error of type apistatus.ObjectNotFound if the requested object is missing in blobovnicza(s).
func (b *BlobStor) GetRangeSmall(prm *GetRangeSmallPrm) (*GetRangeSmallRes, error) {
func (b *BlobStor) GetRangeSmall(prm GetRangeSmallPrm) (*GetRangeSmallRes, error) {
return b.blobovniczas.getRange(prm)
}

View File

@ -20,6 +20,6 @@ type GetSmallRes struct {
// did not allow to completely read the object.
//
// Returns an error of type apistatus.ObjectNotFound if the requested object is missing in blobovnicza(s).
func (b *BlobStor) GetSmall(prm *GetSmallPrm) (*GetSmallRes, error) {
func (b *BlobStor) GetSmall(prm GetSmallPrm) (*GetSmallRes, error) {
return b.blobovniczas.get(prm)
}

View File

@ -88,7 +88,9 @@ func (b *BlobStor) Iterate(prm IteratePrm) (*IterateRes, error) {
elem.blzID = nil
err = b.fsTree.Iterate(new(fstree.IterationPrm).WithHandler(func(_ oid.Address, data []byte) error {
var fsPrm fstree.IterationPrm
fsPrm.WithIgnoreErrors(prm.ignoreErrors)
fsPrm.WithHandler(func(_ oid.Address, data []byte) error {
// decompress the data
elem.data, err = b.decompressor(data)
if err != nil {
@ -99,7 +101,9 @@ func (b *BlobStor) Iterate(prm IteratePrm) (*IterateRes, error) {
}
return prm.handler(elem)
}).WithIgnoreErrors(prm.ignoreErrors))
})
err = b.fsTree.Iterate(fsPrm)
if err != nil {
return nil, fmt.Errorf("fs tree iterator failure: %w", err)

View File

@ -30,7 +30,7 @@ type PutRes struct {
//
// Returns any error encountered that
// did not allow to completely save the object.
func (b *BlobStor) Put(prm *PutPrm) (*PutRes, error) {
func (b *BlobStor) Put(prm PutPrm) (*PutRes, error) {
// marshal object
data, err := prm.obj.Marshal()
if err != nil {

View File

@ -34,8 +34,8 @@ func (s *Shard) Delete(prm DeletePrm) (*DeleteRes, error) {
}
ln := len(prm.addr)
delSmallPrm := new(blobstor.DeleteSmallPrm)
delBigPrm := new(blobstor.DeleteBigPrm)
var delSmallPrm blobstor.DeleteSmallPrm
var delBigPrm blobstor.DeleteBigPrm
smalls := make(map[oid.Address]*blobovnicza.ID, ln)

View File

@ -43,7 +43,7 @@ func (s *Shard) Exists(prm ExistsPrm) (*ExistsRes, error) {
// If the shard is in degraded mode, try to consult blobstor directly.
// Otherwise, just return an error.
if s.GetMode() == ModeDegraded {
p := new(blobstor.ExistsPrm)
var p blobstor.ExistsPrm
p.SetAddress(prm.addr)
res, bErr := s.blobStor.Exists(p)

View File

@ -67,7 +67,7 @@ func (s *Shard) Get(prm GetPrm) (*GetRes, error) {
var big, small storFetcher
big = func(stor *blobstor.BlobStor, _ *blobovnicza.ID) (*objectSDK.Object, error) {
getBigPrm := new(blobstor.GetBigPrm)
var getBigPrm blobstor.GetBigPrm
getBigPrm.SetAddress(prm.addr)
res, err := stor.GetBig(getBigPrm)
@ -79,7 +79,7 @@ func (s *Shard) Get(prm GetPrm) (*GetRes, error) {
}
small = func(stor *blobstor.BlobStor, id *blobovnicza.ID) (*objectSDK.Object, error) {
getSmallPrm := new(blobstor.GetSmallPrm)
var getSmallPrm blobstor.GetSmallPrm
getSmallPrm.SetAddress(prm.addr)
getSmallPrm.SetBlobovniczaID(id)

View File

@ -35,7 +35,7 @@ func (s *Shard) Put(prm PutPrm) (*PutRes, error) {
return nil, ErrReadOnlyMode
}
putPrm := new(blobstor.PutPrm) // form Put parameters
var putPrm blobstor.PutPrm // form Put parameters
putPrm.SetObject(prm.obj)
// exist check are not performed there, these checks should be executed

View File

@ -74,7 +74,7 @@ func (s *Shard) GetRange(prm RngPrm) (*RngRes, error) {
rng.SetLength(prm.ln)
big = func(stor *blobstor.BlobStor, _ *blobovnicza.ID) (*object.Object, error) {
getRngBigPrm := new(blobstor.GetRangeBigPrm)
var getRngBigPrm blobstor.GetRangeBigPrm
getRngBigPrm.SetAddress(prm.addr)
getRngBigPrm.SetRange(rng)
@ -90,7 +90,7 @@ func (s *Shard) GetRange(prm RngPrm) (*RngRes, error) {
}
small = func(stor *blobstor.BlobStor, id *blobovnicza.ID) (*object.Object, error) {
getRngSmallPrm := new(blobstor.GetRangeSmallPrm)
var getRngSmallPrm blobstor.GetRangeSmallPrm
getRngSmallPrm.SetAddress(prm.addr)
getRngSmallPrm.SetRange(rng)
getRngSmallPrm.SetBlobovniczaID(id)

View File

@ -133,7 +133,9 @@ func (c *cache) flushBigObjects() {
}
evictNum := 0
_ = c.fsTree.Iterate(new(fstree.IterationPrm).WithHandler(func(addr oid.Address, data []byte) error {
var prm fstree.IterationPrm
prm.WithHandler(func(addr oid.Address, data []byte) error {
sAddr := addr.EncodeToString()
if _, ok := c.store.flushed.Peek(sAddr); ok {
@ -161,7 +163,9 @@ func (c *cache) flushBigObjects() {
evictNum++
return nil
}))
})
_ = c.fsTree.Iterate(prm)
// evict objects which were successfully written to BlobStor
c.evictObjects(evictNum)
@ -215,8 +219,9 @@ func (c *cache) writeObject(obj *object.Object, metaOnly bool) error {
var id *blobovnicza.ID
if !metaOnly {
prm := new(blobstor.PutPrm)
var prm blobstor.PutPrm
prm.SetObject(obj)
res, err := c.blobstor.Put(prm)
if err != nil {
return err

View File

@ -51,12 +51,16 @@ func (c *cache) Iterate(prm IterationPrm) error {
return err
}
return c.fsTree.Iterate(new(fstree.IterationPrm).WithHandler(func(addr oid.Address, data []byte) error {
var fsPrm fstree.IterationPrm
fsPrm.WithIgnoreErrors(prm.ignoreErrors)
fsPrm.WithHandler(func(addr oid.Address, data []byte) error {
if _, ok := c.flushed.Peek(addr.EncodeToString()); ok {
return nil
}
return prm.handler(data)
}).WithIgnoreErrors(prm.ignoreErrors))
})
return c.fsTree.Iterate(fsPrm)
}
// IterateDB iterates over all objects stored in bbolt.DB instance and passes them to f until error return.