TrueCloudLab/frostfs-node
20
1
Fork 27
Code Issues 105 Pull requests 13 Projects Releases 2 Activity Actions

WIP: Rever writecache improvements #1609

Draft
dstepanov-yadro wants to merge 10 commits from dstepanov-yadro/frostfs-node:fix/revert_writecache_improvements into support/v0.44
pull from: dstepanov-yadro/frostfs-node:fix/revert_writecache_improvements
merge into: TrueCloudLab:support/v0.44
Conversation 0 Commits 10 Files changed 36 +48 -93
7 changed files with 48 additions and 93 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files
Showing only changes of commit 1fc95f6ed8 - Show all commits

1
pkg/local_object_storage/blobstor/common/delete.go
View file

@ -8,7 +8,6 @@ import (
type DeletePrm struct {
Address oid.Address
StorageID []byte
Size uint64
}
// DeleteRes groups the resulting values of Delete operation.

2
pkg/local_object_storage/blobstor/fstree/fstree.go
View file

@ -338,7 +338,7 @@ func (t *FSTree) Delete(ctx context.Context, prm common.DeletePrm) (common.Delet
}
p := t.treePath(prm.Address)
err = t.writer.removeFile(p, prm.Size)
err = t.writer.removeFile(p)
return common.DeleteRes{}, err
}

84
pkg/local_object_storage/blobstor/fstree/fstree_test.go
View file

@ -68,70 +68,34 @@ func TestObjectCounter(t *testing.T) {
var delPrm common.DeletePrm
delPrm.Address = addr
t.Run("without size hint", func(t *testing.T) {
eg, egCtx := errgroup.WithContext(context.Background())
eg, egCtx := errgroup.WithContext(context.Background())
eg.Go(func() error {
for range 1_000 {
_, err := fst.Put(egCtx, putPrm)
if err != nil {
return err
}
eg.Go(func() error {
for range 1_000 {
_, err := fst.Put(egCtx, putPrm)
if err != nil {
return err
}
return nil
})
eg.Go(func() error {
var le logicerr.Logical
for range 1_000 {
_, err := fst.Delete(egCtx, delPrm)
if err != nil && !errors.As(err, &le) {
return err
}
}
return nil
})
require.NoError(t, eg.Wait())
count, size = counter.CountSize()
realCount, realSize, err := fst.countFiles()
require.NoError(t, err)
require.Equal(t, realCount, count, "real %d, actual %d", realCount, count)
require.Equal(t, realSize, size, "real %d, actual %d", realSize, size)
}
return nil
})
t.Run("with size hint", func(t *testing.T) {
delPrm.Size = uint64(len(putPrm.RawData))
eg, egCtx := errgroup.WithContext(context.Background())
eg.Go(func() error {
for range 1_000 {
_, err := fst.Put(egCtx, putPrm)
if err != nil {
return err
}
eg.Go(func() error {
var le logicerr.Logical
for range 1_000 {
_, err := fst.Delete(egCtx, delPrm)
if err != nil && !errors.As(err, &le) {
return err
}
return nil
})
eg.Go(func() error {
var le logicerr.Logical
for range 1_000 {
_, err := fst.Delete(egCtx, delPrm)
if err != nil && !errors.As(err, &le) {
return err
}
}
return nil
})
require.NoError(t, eg.Wait())
count, size = counter.CountSize()
realCount, realSize, err := fst.countFiles()
require.NoError(t, err)
require.Equal(t, realCount, count, "real %d, actual %d", realCount, count)
require.Equal(t, realSize, size, "real %d, actual %d", realSize, size)
}
return nil
})
require.NoError(t, eg.Wait())
count, size = counter.CountSize()
realCount, realSize, err := fst.countFiles()
require.NoError(t, err)
require.Equal(t, realCount, count, "real %d, actual %d", realCount, count)
require.Equal(t, realSize, size, "real %d, actual %d", realSize, size)
}

19
pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go
View file

@ -16,7 +16,7 @@ import (
type writer interface {
writeData(string, []byte) error
removeFile(string, uint64) error
removeFile(string) error
}
type genericWriter struct {
@ -107,10 +107,10 @@ func (w *genericWriter) writeFile(p string, data []byte) error {
return err
}
func (w *genericWriter) removeFile(p string, size uint64) error {
func (w *genericWriter) removeFile(p string) error {
var err error
if w.fileCounterEnabled {
err = w.removeWithCounter(p, size)
err = w.removeWithCounter(p)
} else {
err = os.Remove(p)
}
@ -121,21 +121,18 @@ func (w *genericWriter) removeFile(p string, size uint64) error {
return err
}
func (w *genericWriter) removeWithCounter(p string, size uint64) error {
func (w *genericWriter) removeWithCounter(p string) error {
w.fileGuard.Lock(p)
defer w.fileGuard.Unlock(p)
if size == 0 {
stat, err := os.Stat(p)
if err != nil {
return err
}
size = uint64(stat.Size())
stat, err := os.Stat(p)
if err != nil {
return err
}
if err := os.Remove(p); err != nil {
return err
}
w.fileCounter.Dec(uint64(size))
w.fileCounter.Dec(uint64(stat.Size()))
return nil
}

27
pkg/local_object_storage/blobstor/fstree/fstree_write_linux.go
View file

@ -91,30 +91,25 @@ func (w *linuxWriter) writeFile(p string, data []byte) error {
return errClose
}
func (w *linuxWriter) removeFile(p string, size uint64) error {
func (w *linuxWriter) removeFile(p string) error {
if w.fileCounterEnabled {
w.fileGuard.Lock(p)
defer w.fileGuard.Unlock(p)
if size == 0 {
var stat unix.Stat_t
err := unix.Stat(p, &stat)
if err != nil {
if err == unix.ENOENT {
return logicerr.Wrap(new(apistatus.ObjectNotFound))
}
return err
}
size = uint64(stat.Size)
}
}
err := unix.Unlink(p)
var stat unix.Stat_t
err := unix.Stat(p, &stat)
if err != nil {
if err == unix.ENOENT {
return logicerr.Wrap(new(apistatus.ObjectNotFound))
}
return err
}
err = unix.Unlink(p)
if err != nil && err == unix.ENOENT {
return logicerr.Wrap(new(apistatus.ObjectNotFound))
}
if err == nil {
w.fileCounter.Dec(uint64(size))
w.fileCounter.Dec(uint64(stat.Size))
}
return err
}

4
pkg/local_object_storage/writecache/flush.go
View file

@ -123,7 +123,7 @@ func (c *cache) flushIfAnObjectExistsWorker(ctx context.Context, objInfo objectI
return
}
c.deleteFromDisk(ctx, objInfo.addr, uint64(len(res.RawData)))
c.deleteFromDisk(ctx, objInfo.addr)
}
func (c *cache) reportFlushError(ctx context.Context, msg string, addr string, err error) {
@ -157,7 +157,7 @@ func (c *cache) flushFSTree(ctx context.Context, ignoreErrors bool) error {
return err
}
c.deleteFromDisk(ctx, e.Address, uint64(len(e.ObjectData)))
c.deleteFromDisk(ctx, e.Address)
return nil
}

4
pkg/local_object_storage/writecache/storage.go
View file

@ -40,8 +40,8 @@ func (c *cache) openStore(mod mode.ComponentMode) error {
return nil
}
func (c *cache) deleteFromDisk(ctx context.Context, addr oid.Address, size uint64) {
_, err := c.fsTree.Delete(ctx, common.DeletePrm{Address: addr, Size: size})
func (c *cache) deleteFromDisk(ctx context.Context, addr oid.Address) {
_, err := c.fsTree.Delete(ctx, common.DeletePrm{Address: addr})
if err != nil && !client.IsErrObjectNotFound(err) {
c.log.Error(ctx, logs.WritecacheCantRemoveObjectFromWritecache, zap.Error(err))
} else if err == nil {