realloc/frostfs-node
1
0
Fork 0
forked from TrueCloudLab/frostfs-node
Code Releases Packages Activity

[#1367] fstree: Add size to file counter

Browse source 
FSTree file counter used by writecache. As writecache has now only one
storage, so it is required to use real object size to get writecache
size more accurate than `count * max_object_size`.

Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
This commit is contained in:
Dmitrii Stepanov 2024-09-10 11:01:30 +03:00
parent 5f6c7cbdb1
commit b142b6f48e
9 changed files with 130 additions and 85 deletions
Download patch file Download diff file Expand all files Collapse all files

3
pkg/local_object_storage/writecache/cache.go
View file

@ -29,6 +29,8 @@ type cache struct {
wg sync.WaitGroup
// fsTree contains big files stored directly on file-system.
fsTree *fstree.FSTree
// counter contains atomic counters for the number of objects stored in cache.
counter *fstree.SimpleCounter
}
// wcStorageType is used for write-cache operations logging.
@ -56,6 +58,7 @@ func New(opts ...Option) Cache {
c := &cache{
flushCh: make(chan objectInfo),
mode: mode.Disabled,
counter: fstree.NewSimpleCounter(),
options: options{
log: &logger.Logger{Logger: zap.NewNop()},

2
pkg/local_object_storage/writecache/options.go
View file

@ -30,8 +30,6 @@ type options struct {
// maxCacheCount is the maximum total count of all object saved in cache.
// 0 (no limit) by default.
maxCacheCount uint64
// objCounters contains atomic counters for the number of objects stored in cache.
objCounters counters
// maxBatchSize is the maximum batch size for the small object database.
maxBatchSize int
// maxBatchDelay is the maximum batch wait time for the small object database.

41
pkg/local_object_storage/writecache/state.go
View file

@ -1,18 +1,10 @@
package writecache
import (
"math"
"sync/atomic"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree"
)
func (c *cache) estimateCacheSize() (uint64, uint64) {
fsCount := c.objCounters.FS()
fsSize := fsCount * c.maxObjectSize
c.metrics.SetEstimateSize(0, fsSize)
c.metrics.SetActualCounters(0, fsCount)
return fsCount, fsSize
count, size := c.counter.CountSize()
c.metrics.SetEstimateSize(0, size)
c.metrics.SetActualCounters(0, count)
return count, size
}
func (c *cache) hasEnoughSpaceFS() bool {
@ -27,31 +19,6 @@ func (c *cache) hasEnoughSpace(objectSize uint64) bool {
return c.maxCacheSize >= size+objectSize
}
var _ fstree.FileCounter = &counters{}
type counters struct {
cFS atomic.Uint64
}
func (x *counters) FS() uint64 {
return x.cFS.Load()
}
// Set implements fstree.ObjectCounter.
func (x *counters) Set(v uint64) {
x.cFS.Store(v)
}
// Inc implements fstree.ObjectCounter.
func (x *counters) Inc() {
x.cFS.Add(1)
}
// Dec implements fstree.ObjectCounter.
func (x *counters) Dec() {
x.cFS.Add(math.MaxUint64)
}
func (c *cache) initCounters() error {
c.estimateCacheSize()
return nil

2
pkg/local_object_storage/writecache/storage.go
View file

@ -30,7 +30,7 @@ func (c *cache) openStore(mod mode.ComponentMode) error {
fstree.WithDepth(1),
fstree.WithDirNameLen(1),
fstree.WithNoSync(c.noSync),
fstree.WithFileCounter(&c.objCounters),
fstree.WithFileCounter(c.counter),
)
if err := c.fsTree.Open(mod); err != nil {
return fmt.Errorf("could not open FSTree: %w", err)