[#536] blobovnicza: Drop cache
Each blobovnicza instance is opened while is in use. Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
This commit is contained in:
parent
b9b86d2ec8
commit
c672f59ab8
16 changed files with 586 additions and 474 deletions
|
@ -3,19 +3,12 @@ package blobovniczatree
|
|||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"sync"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"git.frostfs.info/TrueCloudLab/hrw"
|
||||
"github.com/hashicorp/golang-lru/v2/simplelru"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// Blobovniczas represents the storage of the "small" objects.
|
||||
|
@ -61,28 +54,8 @@ import (
|
|||
type Blobovniczas struct {
|
||||
cfg
|
||||
|
||||
// cache of opened filled Blobovniczas
|
||||
opened *simplelru.LRU[string, *blobovnicza.Blobovnicza]
|
||||
// lruMtx protects opened cache.
|
||||
// It isn't RWMutex because `Get` calls must
|
||||
// lock this mutex on write, as LRU info is updated.
|
||||
// It must be taken after activeMtx in case when eviction is possible
|
||||
// i.e. `Add`, `Purge` and `Remove` calls.
|
||||
lruMtx sync.Mutex
|
||||
|
||||
// mutex to exclude parallel bbolt.Open() calls
|
||||
// bbolt.Open() deadlocks if it tries to open already opened file
|
||||
openMtx sync.Mutex
|
||||
|
||||
// list of active (opened, non-filled) Blobovniczas
|
||||
activeMtx sync.RWMutex
|
||||
active map[string]blobovniczaWithIndex
|
||||
}
|
||||
|
||||
type blobovniczaWithIndex struct {
|
||||
ind uint64
|
||||
|
||||
blz *blobovnicza.Blobovnicza
|
||||
commondbManager *dbManager
|
||||
activeDBManager *activeDBManager
|
||||
}
|
||||
|
||||
var _ common.Storage = (*Blobovniczas)(nil)
|
||||
|
@ -102,120 +75,12 @@ func NewBlobovniczaTree(opts ...Option) (blz *Blobovniczas) {
|
|||
blz.blzLeafWidth = blz.blzShallowWidth
|
||||
}
|
||||
|
||||
cache, err := simplelru.NewLRU[string, *blobovnicza.Blobovnicza](blz.openedCacheSize, func(p string, value *blobovnicza.Blobovnicza) {
|
||||
lvlPath := filepath.Dir(p)
|
||||
if b, ok := blz.active[lvlPath]; ok && b.ind == u64FromHexString(filepath.Base(p)) {
|
||||
// This branch is taken if we have recently updated active blobovnicza and remove
|
||||
// it from opened cache.
|
||||
return
|
||||
} else if err := value.Close(); err != nil {
|
||||
blz.log.Error(logs.BlobovniczatreeCouldNotCloseBlobovnicza,
|
||||
zap.String("id", p),
|
||||
zap.String("error", err.Error()),
|
||||
)
|
||||
} else {
|
||||
blz.log.Debug(logs.BlobovniczatreeBlobovniczaSuccessfullyClosedOnEvict,
|
||||
zap.String("id", p),
|
||||
)
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
// occurs only if the size is not positive
|
||||
panic(fmt.Errorf("could not create LRU cache of size %d: %w", blz.openedCacheSize, err))
|
||||
}
|
||||
|
||||
activeMapCapacity := uint64(1)
|
||||
for i := uint64(0); i < blz.blzShallowDepth; i++ {
|
||||
if i+1 == blz.blzShallowDepth {
|
||||
activeMapCapacity *= blz.blzLeafWidth
|
||||
} else {
|
||||
activeMapCapacity *= blz.blzShallowWidth
|
||||
}
|
||||
}
|
||||
|
||||
blz.opened = cache
|
||||
blz.active = make(map[string]blobovniczaWithIndex, activeMapCapacity)
|
||||
blz.commondbManager = newDBManager(blz.rootPath, blz.blzOpts, blz.blzLeafWidth, blz.readOnly, blz.metrics.Blobovnicza(), blz.log)
|
||||
blz.activeDBManager = newActiveDBManager(blz.commondbManager, blz.blzLeafWidth)
|
||||
|
||||
return blz
|
||||
}
|
||||
|
||||
// activates and returns activated blobovnicza of p-level (dir).
|
||||
//
|
||||
// returns error if blobvnicza could not be activated.
|
||||
func (b *Blobovniczas) getActivated(lvlPath string) (blobovniczaWithIndex, error) {
|
||||
return b.updateAndGet(lvlPath, nil)
|
||||
}
|
||||
|
||||
// updates active blobovnicza of p-level (dir).
|
||||
//
|
||||
// if current active blobovnicza's index is not old, it remains unchanged.
|
||||
func (b *Blobovniczas) updateActive(lvlPath string, old *uint64) error {
|
||||
b.log.Debug(logs.BlobovniczatreeUpdatingActiveBlobovnicza, zap.String("path", lvlPath))
|
||||
|
||||
_, err := b.updateAndGet(lvlPath, old)
|
||||
|
||||
b.log.Debug(logs.BlobovniczatreeActiveBlobovniczaSuccessfullyUpdated, zap.String("path", lvlPath))
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// updates and returns active blobovnicza of p-level (dir).
|
||||
//
|
||||
// if current active blobovnicza's index is not old, it is returned unchanged.
|
||||
func (b *Blobovniczas) updateAndGet(lvlPath string, old *uint64) (blobovniczaWithIndex, error) {
|
||||
b.activeMtx.RLock()
|
||||
active, ok := b.active[lvlPath]
|
||||
b.activeMtx.RUnlock()
|
||||
|
||||
if ok {
|
||||
if old != nil {
|
||||
if active.ind == b.blzLeafWidth-1 {
|
||||
return active, logicerr.New("no more Blobovniczas")
|
||||
} else if active.ind != *old {
|
||||
// sort of CAS in order to control concurrent
|
||||
// updateActive calls
|
||||
return active, nil
|
||||
}
|
||||
} else {
|
||||
return active, nil
|
||||
}
|
||||
|
||||
active.ind++
|
||||
}
|
||||
|
||||
var err error
|
||||
if active.blz, err = b.openBlobovnicza(filepath.Join(lvlPath, u64ToHexString(active.ind))); err != nil {
|
||||
return active, err
|
||||
}
|
||||
|
||||
b.activeMtx.Lock()
|
||||
defer b.activeMtx.Unlock()
|
||||
|
||||
// check 2nd time to find out if it blobovnicza was activated while thread was locked
|
||||
tryActive, ok := b.active[lvlPath]
|
||||
if ok && tryActive.blz == active.blz {
|
||||
return tryActive, nil
|
||||
}
|
||||
|
||||
// Remove from opened cache (active blobovnicza should always be opened).
|
||||
// Because `onEvict` callback is called in `Remove`, we need to update
|
||||
// active map beforehand.
|
||||
b.active[lvlPath] = active
|
||||
|
||||
activePath := filepath.Join(lvlPath, u64ToHexString(active.ind))
|
||||
b.lruMtx.Lock()
|
||||
b.opened.Remove(activePath)
|
||||
if ok {
|
||||
b.opened.Add(filepath.Join(lvlPath, u64ToHexString(tryActive.ind)), tryActive.blz)
|
||||
}
|
||||
b.lruMtx.Unlock()
|
||||
|
||||
b.log.Debug(logs.BlobovniczatreeBlobovniczaSuccessfullyActivated,
|
||||
zap.String("path", activePath))
|
||||
|
||||
return active, nil
|
||||
}
|
||||
|
||||
// returns hash of the object address.
|
||||
func addressHash(addr *oid.Address, path string) uint64 {
|
||||
var a string
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue