[#1869] shard: Restore shard mode on failed reloads

Signed-off-by: Evgenii Stratonikov <evgeniy@morphbits.ru>

pkg/local_object_storage/metabase/control.go

@@ -171,7 +171,7 @@ func (db *DB) Reload(opts ...Option) (bool, error) {
 	db.modeMtx.Lock()
 	defer db.modeMtx.Unlock()
 
-	if c.info.Path != "" && filepath.Clean(db.info.Path) != filepath.Clean(c.info.Path) {
+	if db.mode.NoMetabase() || c.info.Path != "" && filepath.Clean(db.info.Path) != filepath.Clean(c.info.Path) {
 		if err := db.Close(); err != nil {
 			return false, err
 		}

pkg/local_object_storage/shard/control.go

@@ -287,12 +287,20 @@ func (s *Shard) Reload(opts ...Option) error {
 		return err
 	}
 	if ok {
+		var err error
 		if c.refillMetabase {
-			return s.refillMetabase()
+			// Here we refill metabase only if a new instance was opened. This is a feature,
+			// we don't want to hang for some time just because we forgot to change
+			// config after the node was updated.
+			err = s.refillMetabase()
 		} else {
-			return s.metaBase.Init()
+			err = s.metaBase.Init()
+		}
+		if err != nil {
+			_ = s.setMode(mode.DegradedReadOnly)
+			return err
 		}
 	}
 
-	return nil
+	return s.setMode(mode.ReadWrite)
 }

pkg/local_object_storage/shard/reload_test.go

@@ -113,6 +113,16 @@ func TestShardReload(t *testing.T) {
 
 			// Old objects are still accessible.
 			checkHasObjects(t, true)
+
+			// Successive reload produces no undesired effects.
+			require.NoError(t, os.RemoveAll(badPath))
+			require.NoError(t, sh.Reload(newOpts...))
+
+			obj = newObject()
+			require.NoError(t, putObject(sh, obj))
+
+			objects = append(objects, objAddr{obj: obj, addr: objectCore.AddressOf(obj)})
+			checkHasObjects(t, true)
 		})
 	})
 }