dstepanov-yadro/frostfs-node
1
0
Fork 0
forked from TrueCloudLab/frostfs-node
Code Issues Pull requests Projects Releases Packages Wiki Activity

[#1745] writecache: Set flush parameters based on max size

Browse source 
Signed-off-by: Evgenii Stratonikov <evgeniy@morphbits.ru>
This commit is contained in:
Evgenii Stratonikov 2022-09-01 09:33:56 +03:00 committed by fyrchik
parent c04126f35e
commit e6cf0b3d42
2 changed files with 11 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
pkg/local_object_storage/writecache/storage.go
View file

@ -20,6 +20,9 @@ import (
// store represents persistent storage with in-memory LRU cache // store represents persistent storage with in-memory LRU cache
// for flushed items on top of it. // for flushed items on top of it.
type store struct { type store struct {
maxFlushedMarksCount int
maxRemoveBatchSize int
flushed simplelru.LRUCache flushed simplelru.LRUCache
db *bbolt.DB db *bbolt.DB
@ -27,11 +30,6 @@ type store struct {
fsKeysToRemove []string fsKeysToRemove []string
} }
const (
maxFlushedMarksCount = 256 * 1024 * 8
maxRemoveBatchSize = maxFlushedMarksCount / 4
)
const dbName = "small.bolt" const dbName = "small.bolt"
func (c *cache) openStore(readOnly bool) error { func (c *cache) openStore(readOnly bool) error {
@ -70,7 +68,7 @@ func (c *cache) openStore(readOnly bool) error {
// Write-cache can be opened multiple times during `SetMode`. // Write-cache can be opened multiple times during `SetMode`.
// flushed map must not be re-created in this case. // flushed map must not be re-created in this case.
if c.flushed == nil { if c.flushed == nil {
c.flushed, _ = lru.NewWithEvict(maxFlushedMarksCount, c.removeFlushed) c.flushed, _ = lru.NewWithEvict(c.maxFlushedMarksCount, c.removeFlushed)
} }
return nil return nil
} }
@ -87,7 +85,7 @@ func (c *cache) removeFlushed(key, value interface{}) {
c.fsKeysToRemove = append(c.fsKeysToRemove, key.(string)) c.fsKeysToRemove = append(c.fsKeysToRemove, key.(string))
} }
if len(c.dbKeysToRemove)+len(c.fsKeysToRemove) >= maxRemoveBatchSize { if len(c.dbKeysToRemove)+len(c.fsKeysToRemove) >= c.maxRemoveBatchSize {
c.dbKeysToRemove = c.deleteFromDB(c.dbKeysToRemove) c.dbKeysToRemove = c.deleteFromDB(c.dbKeysToRemove)
c.fsKeysToRemove = c.deleteFromDisk(c.fsKeysToRemove) c.fsKeysToRemove = c.deleteFromDisk(c.fsKeysToRemove)
} }

6
pkg/local_object_storage/writecache/writecache.go
View file

@ -97,6 +97,12 @@ func New(opts ...Option) Cache {
opts[i](&c.options) opts[i](&c.options)
} }
// Make the LRU cache contain which take approximately 3/4 of the maximum space.
// Assume small and big objects are stored in 50-50 proportion.
c.maxFlushedMarksCount = int(c.maxCacheSize/c.maxObjectSize+c.maxCacheSize/c.smallObjectSize) / 2 * 3 / 4
// Trigger the removal when the cache is 7/8 full, so that new items can still arrive.
c.maxRemoveBatchSize = c.maxFlushedMarksCount / 8
return c return c
} }