[#585] fstree: Add optional file counter
Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
This commit is contained in:
parent
baad49990c
commit
58c8722c81
12 changed files with 312 additions and 172 deletions
|
@ -2,20 +2,24 @@ package writecachebbolt
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree"
|
||||
"go.etcd.io/bbolt"
|
||||
"go.uber.org/zap"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
func (c *cache) estimateCacheSize() uint64 {
|
||||
db := c.objCounters.DB() * c.smallObjectSize
|
||||
fstree := c.objCounters.FS() * c.maxObjectSize
|
||||
c.metrics.SetEstimateSize(db, fstree)
|
||||
return db + fstree
|
||||
dbCount := c.objCounters.DB()
|
||||
fsCount := c.objCounters.FS()
|
||||
if fsCount > 0 {
|
||||
fsCount-- //db file
|
||||
}
|
||||
dbSize := dbCount * c.smallObjectSize
|
||||
fsSize := fsCount * c.maxObjectSize
|
||||
c.metrics.SetEstimateSize(dbSize, fsSize)
|
||||
c.metrics.SetActualCounters(dbCount, fsCount)
|
||||
return dbSize + fsSize
|
||||
}
|
||||
|
||||
func (c *cache) incSizeDB(sz uint64) uint64 {
|
||||
|
@ -26,6 +30,8 @@ func (c *cache) incSizeFS(sz uint64) uint64 {
|
|||
return sz + c.maxObjectSize
|
||||
}
|
||||
|
||||
var _ fstree.FileCounter = &counters{}
|
||||
|
||||
type counters struct {
|
||||
cDB, cFS atomic.Uint64
|
||||
}
|
||||
|
@ -38,60 +44,33 @@ func (x *counters) FS() uint64 {
|
|||
return x.cFS.Load()
|
||||
}
|
||||
|
||||
func (c *cache) setCounters() error {
|
||||
// Set implements fstree.ObjectCounter.
|
||||
func (x *counters) Set(v uint64) {
|
||||
x.cFS.Store(v)
|
||||
}
|
||||
|
||||
// Inc implements fstree.ObjectCounter.
|
||||
func (x *counters) Inc() {
|
||||
x.cFS.Add(1)
|
||||
}
|
||||
|
||||
// Dec implements fstree.ObjectCounter.
|
||||
func (x *counters) Dec() {
|
||||
x.cFS.Add(math.MaxUint64)
|
||||
}
|
||||
|
||||
func (c *cache) initCounters() error {
|
||||
var inDB uint64
|
||||
var inFS uint64
|
||||
|
||||
var eg errgroup.Group
|
||||
|
||||
eg.Go(func() error {
|
||||
err := c.db.View(func(tx *bbolt.Tx) error {
|
||||
b := tx.Bucket(defaultBucket)
|
||||
if b != nil {
|
||||
inDB = uint64(b.Stats().KeyN)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not read write-cache DB counter: %w", err)
|
||||
err := c.db.View(func(tx *bbolt.Tx) error {
|
||||
b := tx.Bucket(defaultBucket)
|
||||
if b != nil {
|
||||
inDB = uint64(b.Stats().KeyN)
|
||||
}
|
||||
c.objCounters.cDB.Store(inDB)
|
||||
return nil
|
||||
})
|
||||
|
||||
eg.Go(func() error {
|
||||
var err error
|
||||
inFS, err = c.fsTree.NumberOfObjects()
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not read write-cache FS counter: %w", err)
|
||||
}
|
||||
if inFS > 0 {
|
||||
inFS-- //small.bolt DB file
|
||||
}
|
||||
c.objCounters.cFS.Store(inFS)
|
||||
return nil
|
||||
})
|
||||
if err := eg.Wait(); err != nil {
|
||||
return err
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not read write-cache DB counter: %w", err)
|
||||
}
|
||||
c.metrics.SetActualCounters(inDB, inFS)
|
||||
c.objCounters.cDB.Store(inDB)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *cache) runDBCounterLoop() {
|
||||
go func() {
|
||||
t := time.NewTicker(time.Second * 30)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-t.C:
|
||||
err := c.setCounters()
|
||||
if err != nil {
|
||||
c.log.Warn(logs.FailedToCountWritecacheItems, zap.Error(err))
|
||||
}
|
||||
case <-c.closeCh:
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue