[#1825] writecache: Flush cache when moving to the DEGRADED mode

Degraded mode allows us to operate without an SSD,
thus writecache should be unavailable in this mode.

Signed-off-by: Evgenii Stratonikov <evgeniy@morphbits.ru>
This commit is contained in:
Evgenii Stratonikov 2022-09-30 13:41:37 +03:00 committed by fyrchik
parent 236414df49
commit 8b3b16fe62
5 changed files with 50 additions and 11 deletions

View file

@ -16,6 +16,7 @@ Changelog for NeoFS Node
### Changed
- Allow to evacuate shard data with `EvacuateShard` control RPC (#1800)
- Flush write-cache when moving shard to DEGRADED mode (#1825)
### Fixed
- Description of command `netmap nodeinfo` (#1821)

View file

@ -49,10 +49,11 @@ func (s *Shard) Put(prm PutPrm) (PutRes, error) {
// exist check are not performed there, these checks should be executed
// ahead of `Put` by storage engine
if s.hasWriteCache() {
tryCache := s.hasWriteCache() && !m.NoMetabase()
if tryCache {
res, err = s.writeCache.Put(putPrm)
}
if err != nil || !s.hasWriteCache() {
if err != nil || !tryCache {
if err != nil {
s.log.Debug("can't put object to the write-cache, trying blobstor",
zap.String("err", err.Error()))

View file

@ -50,7 +50,7 @@ func (c *cache) runFlushLoop() {
for {
select {
case <-tt.C:
c.flush()
c.flushDB()
tt.Reset(defaultFlushInterval)
case <-c.closeCh:
return
@ -59,7 +59,7 @@ func (c *cache) runFlushLoop() {
}()
}
func (c *cache) flush() {
func (c *cache) flushDB() {
lastKey := []byte{}
var m []objectInfo
for {
@ -241,6 +241,10 @@ func (c *cache) Flush(ignoreErrors bool) error {
return errMustBeReadOnly
}
return c.flush(ignoreErrors)
}
func (c *cache) flush(ignoreErrors bool) error {
var prm common.IteratePrm
prm.IgnoreErrors = ignoreErrors
prm.LazyHandler = func(addr oid.Address, f func() ([]byte, error)) error {

View file

@ -109,10 +109,6 @@ func TestFlush(t *testing.T) {
wc, bs, mb := newCache(t)
objects := putObjects(t, wc)
t.Run("must be read-only", func(t *testing.T) {
require.ErrorIs(t, wc.Flush(false), errMustBeReadOnly)
})
require.NoError(t, wc.SetMode(mode.ReadOnly))
require.NoError(t, bs.SetMode(mode.ReadWrite))
require.NoError(t, mb.SetMode(mode.ReadWrite))
@ -135,6 +131,36 @@ func TestFlush(t *testing.T) {
check(t, mb, bs, objects[2:])
})
t.Run("flush on moving to degraded mode", func(t *testing.T) {
wc, bs, mb := newCache(t)
objects := putObjects(t, wc)
// Blobstor is read-only, so we expect en error from `flush` here.
require.Error(t, wc.SetMode(mode.Degraded))
// First move to read-only mode to close background workers.
require.NoError(t, wc.SetMode(mode.ReadOnly))
require.NoError(t, bs.SetMode(mode.ReadWrite))
require.NoError(t, mb.SetMode(mode.ReadWrite))
wc.(*cache).flushed.Add(objects[0].addr.EncodeToString(), true)
wc.(*cache).flushed.Add(objects[1].addr.EncodeToString(), false)
require.NoError(t, wc.SetMode(mode.Degraded))
for i := 0; i < 2; i++ {
var mPrm meta.GetPrm
mPrm.SetAddress(objects[i].addr)
_, err := mb.Get(mPrm)
require.Error(t, err)
_, err = bs.Get(common.GetPrm{Address: objects[i].addr})
require.Error(t, err)
}
check(t, mb, bs, objects[2:])
})
t.Run("ignore errors", func(t *testing.T) {
testIgnoreErrors := func(t *testing.T, f func(*cache)) {
wc, bs, mb := newCache(t)

View file

@ -18,9 +18,11 @@ func (c *cache) SetMode(m mode.Mode) error {
c.modeMtx.Lock()
defer c.modeMtx.Unlock()
if m.ReadOnly() == c.readOnly() {
c.mode = m
return nil
if m.NoMetabase() && !c.mode.NoMetabase() {
err := c.flush(true)
if err != nil {
return err
}
}
if c.db != nil {
@ -37,6 +39,11 @@ func (c *cache) SetMode(m mode.Mode) error {
time.Sleep(time.Second)
}
if m.NoMetabase() {
c.mode = m
return nil
}
if err := c.openStore(m.ReadOnly()); err != nil {
return err
}