2020-11-30 14:30:49 +00:00
|
|
|
package shard
|
|
|
|
|
|
|
|
import (
|
2023-03-23 14:59:14 +00:00
|
|
|
"context"
|
2022-07-27 18:38:28 +00:00
|
|
|
"errors"
|
2021-05-18 08:12:51 +00:00
|
|
|
"fmt"
|
2024-03-12 14:36:26 +00:00
|
|
|
"sync"
|
2021-09-13 13:38:58 +00:00
|
|
|
|
2023-04-12 14:35:10 +00:00
|
|
|
"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
|
2023-03-07 13:38:26 +00:00
|
|
|
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
|
|
|
|
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor"
|
|
|
|
meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase"
|
|
|
|
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
|
2023-05-31 09:24:04 +00:00
|
|
|
"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
|
2023-08-04 11:14:07 +00:00
|
|
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
|
2023-03-07 13:38:26 +00:00
|
|
|
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
|
|
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
2022-06-21 07:52:03 +00:00
|
|
|
"go.uber.org/zap"
|
2024-03-05 12:39:50 +00:00
|
|
|
"golang.org/x/sync/errgroup"
|
2020-11-30 14:30:49 +00:00
|
|
|
)
|
|
|
|
|
2022-06-27 08:06:01 +00:00
|
|
|
func (s *Shard) handleMetabaseFailure(stage string, err error) error {
|
2023-04-12 14:35:10 +00:00
|
|
|
s.log.Error(logs.ShardMetabaseFailureSwitchingMode,
|
2022-06-27 08:06:01 +00:00
|
|
|
zap.String("stage", stage),
|
|
|
|
zap.Stringer("mode", mode.ReadOnly),
|
|
|
|
zap.Error(err))
|
|
|
|
|
|
|
|
err = s.SetMode(mode.ReadOnly)
|
|
|
|
if err == nil {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2023-04-12 14:35:10 +00:00
|
|
|
s.log.Error(logs.ShardCantMoveShardToReadonlySwitchMode,
|
2022-06-27 08:06:01 +00:00
|
|
|
zap.String("stage", stage),
|
|
|
|
zap.Stringer("mode", mode.DegradedReadOnly),
|
|
|
|
zap.Error(err))
|
|
|
|
|
|
|
|
err = s.SetMode(mode.DegradedReadOnly)
|
|
|
|
if err != nil {
|
2024-06-04 13:28:47 +00:00
|
|
|
return fmt.Errorf("could not switch to mode %s", mode.Mode(mode.DegradedReadOnly))
|
2022-06-27 08:06:01 +00:00
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2020-11-30 14:30:49 +00:00
|
|
|
// Open opens all Shard's components.
|
2023-08-31 16:26:47 +00:00
|
|
|
func (s *Shard) Open(ctx context.Context) error {
|
|
|
|
components := []interface {
|
2024-02-09 06:17:17 +00:00
|
|
|
Open(context.Context, mode.Mode) error
|
2023-08-31 16:26:47 +00:00
|
|
|
}{
|
2024-02-09 06:17:17 +00:00
|
|
|
s.blobStor,
|
2022-07-18 10:16:23 +00:00
|
|
|
}
|
2024-02-09 06:17:17 +00:00
|
|
|
m := s.GetMode()
|
2022-07-18 10:16:23 +00:00
|
|
|
|
2024-02-09 06:17:17 +00:00
|
|
|
if !m.NoMetabase() {
|
|
|
|
components = append(components, s.metaBase)
|
|
|
|
}
|
|
|
|
|
|
|
|
if s.hasWriteCache() && !m.NoMetabase() {
|
2020-12-01 07:35:25 +00:00
|
|
|
components = append(components, s.writeCache)
|
|
|
|
}
|
|
|
|
|
2022-06-28 13:42:50 +00:00
|
|
|
if s.pilorama != nil {
|
|
|
|
components = append(components, s.pilorama)
|
|
|
|
}
|
|
|
|
|
2022-08-22 12:25:26 +00:00
|
|
|
for i, component := range components {
|
2024-02-09 06:17:17 +00:00
|
|
|
if err := component.Open(ctx, m); err != nil {
|
2022-06-27 08:06:01 +00:00
|
|
|
if component == s.metaBase {
|
2022-08-22 12:25:26 +00:00
|
|
|
// We must first open all other components to avoid
|
|
|
|
// opening non-existent DB in read-only mode.
|
|
|
|
for j := i + 1; j < len(components); j++ {
|
2024-02-09 06:17:17 +00:00
|
|
|
if err := components[j].Open(ctx, m); err != nil {
|
2022-08-22 12:25:26 +00:00
|
|
|
// Other components must be opened, fail.
|
|
|
|
return fmt.Errorf("could not open %T: %w", components[j], err)
|
|
|
|
}
|
|
|
|
}
|
2022-06-27 08:06:01 +00:00
|
|
|
err = s.handleMetabaseFailure("open", err)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
break
|
|
|
|
}
|
|
|
|
|
2021-05-18 08:12:51 +00:00
|
|
|
return fmt.Errorf("could not open %T: %w", component, err)
|
2020-11-30 14:30:49 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2022-06-27 08:06:01 +00:00
|
|
|
type metabaseSynchronizer Shard
|
|
|
|
|
|
|
|
func (x *metabaseSynchronizer) Init() error {
|
2023-04-12 14:01:29 +00:00
|
|
|
ctx, span := tracing.StartSpanFromContext(context.TODO(), "metabaseSynchronizer.Init")
|
|
|
|
defer span.End()
|
|
|
|
|
|
|
|
return (*Shard)(x).refillMetabase(ctx)
|
2022-06-27 08:06:01 +00:00
|
|
|
}
|
|
|
|
|
2020-11-30 14:30:49 +00:00
|
|
|
// Init initializes all Shard's components.
|
2023-03-23 14:59:14 +00:00
|
|
|
func (s *Shard) Init(ctx context.Context) error {
|
2024-07-31 13:33:31 +00:00
|
|
|
m := s.GetMode()
|
|
|
|
if err := s.initializeComponents(m); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
s.updateMetrics(ctx)
|
|
|
|
|
|
|
|
s.gc = &gc{
|
|
|
|
gcCfg: &s.gcCfg,
|
|
|
|
remover: s.removeGarbage,
|
|
|
|
stopChannel: make(chan struct{}),
|
|
|
|
eventChan: make(chan Event),
|
|
|
|
mEventHandler: map[eventType]*eventHandlers{
|
|
|
|
eventNewEpoch: {
|
|
|
|
cancelFunc: func() {},
|
|
|
|
handlers: []eventHandler{
|
|
|
|
s.collectExpiredLocks,
|
|
|
|
s.collectExpiredObjects,
|
|
|
|
s.collectExpiredTombstones,
|
|
|
|
s.collectExpiredMetrics,
|
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
}
|
|
|
|
if s.gc.metrics != nil {
|
|
|
|
s.gc.metrics.SetShardID(s.info.ID.String())
|
|
|
|
}
|
|
|
|
|
|
|
|
s.gc.init(ctx)
|
|
|
|
|
|
|
|
s.rb = newRebuilder(s.rebuildLimiter)
|
|
|
|
if !m.NoMetabase() {
|
|
|
|
s.rb.Start(ctx, s.blobStor, s.metaBase, s.log)
|
|
|
|
}
|
|
|
|
s.writecacheSealCancel.Store(dummyCancel)
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (s *Shard) initializeComponents(m mode.Mode) error {
|
2022-06-27 08:06:01 +00:00
|
|
|
type initializer interface {
|
|
|
|
Init() error
|
2021-09-13 13:56:07 +00:00
|
|
|
}
|
|
|
|
|
2022-06-27 08:06:01 +00:00
|
|
|
var components []initializer
|
|
|
|
|
2024-02-09 06:17:17 +00:00
|
|
|
if !m.NoMetabase() {
|
2022-06-27 08:06:01 +00:00
|
|
|
var initMetabase initializer
|
|
|
|
|
2023-06-22 07:46:56 +00:00
|
|
|
if s.NeedRefillMetabase() {
|
2022-06-27 08:06:01 +00:00
|
|
|
initMetabase = (*metabaseSynchronizer)(s)
|
|
|
|
} else {
|
|
|
|
initMetabase = s.metaBase
|
|
|
|
}
|
|
|
|
|
|
|
|
components = []initializer{
|
|
|
|
s.blobStor, initMetabase,
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
components = []initializer{s.blobStor}
|
2022-07-18 10:16:23 +00:00
|
|
|
}
|
|
|
|
|
2024-02-09 06:17:17 +00:00
|
|
|
if s.hasWriteCache() && !m.NoMetabase() {
|
2022-06-27 08:06:01 +00:00
|
|
|
components = append(components, s.writeCache)
|
2020-12-01 07:35:25 +00:00
|
|
|
}
|
|
|
|
|
2022-06-28 13:42:50 +00:00
|
|
|
if s.pilorama != nil {
|
2022-06-27 08:06:01 +00:00
|
|
|
components = append(components, s.pilorama)
|
2022-06-28 13:42:50 +00:00
|
|
|
}
|
|
|
|
|
2020-12-01 07:35:25 +00:00
|
|
|
for _, component := range components {
|
2022-06-27 08:06:01 +00:00
|
|
|
if err := component.Init(); err != nil {
|
|
|
|
if component == s.metaBase {
|
2024-08-29 13:33:30 +00:00
|
|
|
if errors.Is(err, meta.ErrOutdatedVersion) || errors.Is(err, meta.ErrIncompletedUpgrade) {
|
2022-10-21 19:03:24 +00:00
|
|
|
return fmt.Errorf("metabase initialization: %w", err)
|
2022-09-21 20:57:46 +00:00
|
|
|
}
|
|
|
|
|
2022-06-27 08:06:01 +00:00
|
|
|
err = s.handleMetabaseFailure("init", err)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
break
|
|
|
|
}
|
|
|
|
|
2021-05-18 08:12:51 +00:00
|
|
|
return fmt.Errorf("could not initialize %T: %w", component, err)
|
2020-11-30 14:30:49 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2023-04-12 14:01:29 +00:00
|
|
|
func (s *Shard) refillMetabase(ctx context.Context) error {
|
2024-03-12 14:36:26 +00:00
|
|
|
path := s.metaBase.DumpInfo().Path
|
|
|
|
s.metricsWriter.SetRefillStatus(path, "running")
|
|
|
|
s.metricsWriter.SetRefillPercent(path, 0)
|
|
|
|
var success bool
|
|
|
|
defer func() {
|
|
|
|
if success {
|
|
|
|
s.metricsWriter.SetRefillStatus(path, "completed")
|
|
|
|
} else {
|
|
|
|
s.metricsWriter.SetRefillStatus(path, "failed")
|
|
|
|
}
|
|
|
|
}()
|
|
|
|
|
2021-09-13 13:38:58 +00:00
|
|
|
err := s.metaBase.Reset()
|
|
|
|
if err != nil {
|
|
|
|
return fmt.Errorf("could not reset metabase: %w", err)
|
|
|
|
}
|
|
|
|
|
2024-03-12 14:36:26 +00:00
|
|
|
withCount := true
|
|
|
|
totalObjects, err := s.blobStor.ObjectsCount(ctx)
|
|
|
|
if err != nil {
|
|
|
|
s.log.Warn(logs.EngineRefillFailedToGetObjectsCount, zap.Error(err))
|
|
|
|
withCount = false
|
|
|
|
}
|
|
|
|
|
2024-03-05 12:39:50 +00:00
|
|
|
eg, egCtx := errgroup.WithContext(ctx)
|
|
|
|
if s.cfg.refillMetabaseWorkersCount > 0 {
|
|
|
|
eg.SetLimit(s.cfg.refillMetabaseWorkersCount)
|
|
|
|
}
|
|
|
|
|
2024-03-12 14:36:26 +00:00
|
|
|
var completedCount uint64
|
|
|
|
var metricGuard sync.Mutex
|
2024-03-05 12:39:50 +00:00
|
|
|
itErr := blobstor.IterateBinaryObjects(egCtx, s.blobStor, func(addr oid.Address, data []byte, descriptor []byte) error {
|
|
|
|
eg.Go(func() error {
|
2024-03-12 14:36:26 +00:00
|
|
|
var success bool
|
|
|
|
defer func() {
|
|
|
|
s.metricsWriter.IncRefillObjectsCount(path, len(data), success)
|
|
|
|
if withCount {
|
|
|
|
metricGuard.Lock()
|
|
|
|
completedCount++
|
|
|
|
s.metricsWriter.SetRefillPercent(path, uint32(completedCount*100/totalObjects))
|
|
|
|
metricGuard.Unlock()
|
|
|
|
}
|
|
|
|
}()
|
2024-03-05 12:39:50 +00:00
|
|
|
|
2024-03-12 14:36:26 +00:00
|
|
|
if err := s.refillObject(egCtx, data, addr, descriptor); err != nil {
|
2024-03-05 12:39:50 +00:00
|
|
|
return err
|
|
|
|
}
|
2024-03-12 14:36:26 +00:00
|
|
|
success = true
|
2022-06-21 07:52:03 +00:00
|
|
|
return nil
|
2024-03-05 12:39:50 +00:00
|
|
|
})
|
2022-06-21 07:52:03 +00:00
|
|
|
|
2024-03-05 12:39:50 +00:00
|
|
|
select {
|
|
|
|
case <-egCtx.Done():
|
|
|
|
return egCtx.Err()
|
2023-03-21 14:06:46 +00:00
|
|
|
default:
|
2024-03-05 12:39:50 +00:00
|
|
|
return nil
|
2023-03-21 14:06:46 +00:00
|
|
|
}
|
2024-03-05 12:39:50 +00:00
|
|
|
})
|
2022-07-12 14:42:55 +00:00
|
|
|
|
2024-03-05 12:39:50 +00:00
|
|
|
egErr := eg.Wait()
|
2021-09-14 13:37:09 +00:00
|
|
|
|
2024-03-05 12:39:50 +00:00
|
|
|
err = errors.Join(egErr, itErr)
|
2022-09-08 17:32:02 +00:00
|
|
|
if err != nil {
|
|
|
|
return fmt.Errorf("could not put objects to the meta: %w", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
err = s.metaBase.SyncCounters()
|
|
|
|
if err != nil {
|
|
|
|
return fmt.Errorf("could not sync object counters: %w", err)
|
|
|
|
}
|
|
|
|
|
2024-03-12 14:36:26 +00:00
|
|
|
success = true
|
|
|
|
s.metricsWriter.SetRefillPercent(path, 100)
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (s *Shard) refillObject(ctx context.Context, data []byte, addr oid.Address, descriptor []byte) error {
|
|
|
|
obj := objectSDK.New()
|
|
|
|
if err := obj.Unmarshal(data); err != nil {
|
|
|
|
s.log.Warn(logs.ShardCouldNotUnmarshalObject,
|
|
|
|
zap.Stringer("address", addr),
|
|
|
|
zap.String("err", err.Error()))
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
var err error
|
|
|
|
switch obj.Type() {
|
|
|
|
case objectSDK.TypeTombstone:
|
|
|
|
err = s.refillTombstoneObject(ctx, obj)
|
|
|
|
case objectSDK.TypeLock:
|
|
|
|
err = s.refillLockObject(ctx, obj)
|
|
|
|
default:
|
|
|
|
}
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
var mPrm meta.PutPrm
|
|
|
|
mPrm.SetObject(obj)
|
|
|
|
mPrm.SetStorageID(descriptor)
|
|
|
|
|
|
|
|
_, err = s.metaBase.Put(ctx, mPrm)
|
|
|
|
if err != nil && !client.IsErrObjectAlreadyRemoved(err) && !errors.Is(err, meta.ErrObjectIsExpired) {
|
|
|
|
return err
|
|
|
|
}
|
2022-09-08 17:32:02 +00:00
|
|
|
return nil
|
2021-09-13 13:38:58 +00:00
|
|
|
}
|
|
|
|
|
2023-04-12 14:01:29 +00:00
|
|
|
func (s *Shard) refillLockObject(ctx context.Context, obj *objectSDK.Object) error {
|
2023-03-21 14:06:46 +00:00
|
|
|
var lock objectSDK.Lock
|
|
|
|
if err := lock.Unmarshal(obj.Payload()); err != nil {
|
|
|
|
return fmt.Errorf("could not unmarshal lock content: %w", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
locked := make([]oid.ID, lock.NumberOfMembers())
|
|
|
|
lock.ReadMembers(locked)
|
|
|
|
|
|
|
|
cnr, _ := obj.ContainerID()
|
|
|
|
id, _ := obj.ID()
|
2023-04-12 14:01:29 +00:00
|
|
|
err := s.metaBase.Lock(ctx, cnr, id, locked)
|
2023-03-21 14:06:46 +00:00
|
|
|
if err != nil {
|
|
|
|
return fmt.Errorf("could not lock objects: %w", err)
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2023-04-12 14:01:29 +00:00
|
|
|
func (s *Shard) refillTombstoneObject(ctx context.Context, obj *objectSDK.Object) error {
|
2023-03-21 14:06:46 +00:00
|
|
|
tombstone := objectSDK.NewTombstone()
|
|
|
|
|
|
|
|
if err := tombstone.Unmarshal(obj.Payload()); err != nil {
|
|
|
|
return fmt.Errorf("could not unmarshal tombstone content: %w", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
tombAddr := object.AddressOf(obj)
|
|
|
|
memberIDs := tombstone.Members()
|
|
|
|
tombMembers := make([]oid.Address, 0, len(memberIDs))
|
|
|
|
|
|
|
|
for i := range memberIDs {
|
|
|
|
a := tombAddr
|
|
|
|
a.SetObject(memberIDs[i])
|
|
|
|
|
|
|
|
tombMembers = append(tombMembers, a)
|
|
|
|
}
|
|
|
|
|
|
|
|
var inhumePrm meta.InhumePrm
|
|
|
|
|
|
|
|
inhumePrm.SetTombstoneAddress(tombAddr)
|
|
|
|
inhumePrm.SetAddresses(tombMembers...)
|
|
|
|
|
2023-04-12 14:01:29 +00:00
|
|
|
_, err := s.metaBase.Inhume(ctx, inhumePrm)
|
2023-03-21 14:06:46 +00:00
|
|
|
if err != nil {
|
|
|
|
return fmt.Errorf("could not inhume objects: %w", err)
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2020-11-30 14:30:49 +00:00
|
|
|
// Close releases all Shard's components.
|
|
|
|
func (s *Shard) Close() error {
|
2023-09-19 10:50:14 +00:00
|
|
|
if s.rb != nil {
|
|
|
|
s.rb.Stop(s.log)
|
|
|
|
}
|
2022-01-18 14:26:30 +00:00
|
|
|
components := []interface{ Close() error }{}
|
2020-12-01 07:35:25 +00:00
|
|
|
|
2022-07-18 10:16:23 +00:00
|
|
|
if s.pilorama != nil {
|
|
|
|
components = append(components, s.pilorama)
|
|
|
|
}
|
|
|
|
|
2022-06-28 13:42:50 +00:00
|
|
|
if s.hasWriteCache() {
|
2024-07-31 13:30:07 +00:00
|
|
|
prev := s.writecacheSealCancel.Swap(notInitializedCancel)
|
|
|
|
prev.cancel() // no need to wait: writecache.Seal and writecache.Close lock the same mutex
|
2022-06-28 13:42:50 +00:00
|
|
|
components = append(components, s.writeCache)
|
|
|
|
}
|
|
|
|
|
2022-07-18 10:16:23 +00:00
|
|
|
components = append(components, s.blobStor, s.metaBase)
|
2022-01-18 14:26:30 +00:00
|
|
|
|
2023-02-06 10:35:28 +00:00
|
|
|
var lastErr error
|
2020-12-01 07:35:25 +00:00
|
|
|
for _, component := range components {
|
2020-11-30 14:30:49 +00:00
|
|
|
if err := component.Close(); err != nil {
|
2023-02-06 10:35:28 +00:00
|
|
|
lastErr = err
|
2023-04-12 14:35:10 +00:00
|
|
|
s.log.Error(logs.ShardCouldNotCloseShardComponent, zap.Error(err))
|
2020-11-30 14:30:49 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-01-18 11:12:35 +00:00
|
|
|
// If Init/Open was unsuccessful gc can be nil.
|
|
|
|
if s.gc != nil {
|
|
|
|
s.gc.stop()
|
|
|
|
}
|
2021-08-04 12:04:37 +00:00
|
|
|
|
2023-02-06 10:35:28 +00:00
|
|
|
return lastErr
|
2020-11-30 14:30:49 +00:00
|
|
|
}
|
2022-10-16 11:39:47 +00:00
|
|
|
|
|
|
|
// Reload reloads configuration portions that are necessary.
|
|
|
|
// If a config option is invalid, it logs an error and returns nil.
|
|
|
|
// If there was a problem with applying new configuration, an error is returned.
|
2023-04-12 14:01:29 +00:00
|
|
|
func (s *Shard) Reload(ctx context.Context, opts ...Option) error {
|
|
|
|
ctx, span := tracing.StartSpanFromContext(ctx, "Shard.Reload")
|
|
|
|
defer span.End()
|
|
|
|
|
2022-10-16 11:39:47 +00:00
|
|
|
// Do not use defaultCfg here missing options need not be reloaded.
|
|
|
|
var c cfg
|
|
|
|
for i := range opts {
|
|
|
|
opts[i](&c)
|
|
|
|
}
|
|
|
|
|
2023-05-23 12:02:38 +00:00
|
|
|
unlock := s.lockExclusive()
|
|
|
|
defer unlock()
|
2022-10-16 11:39:47 +00:00
|
|
|
|
2023-09-19 10:50:14 +00:00
|
|
|
s.rb.Stop(s.log)
|
2024-02-14 12:39:28 +00:00
|
|
|
if !s.info.Mode.NoMetabase() {
|
|
|
|
defer func() {
|
|
|
|
s.rb.Start(ctx, s.blobStor, s.metaBase, s.log)
|
|
|
|
}()
|
|
|
|
}
|
2023-09-19 10:50:14 +00:00
|
|
|
|
2022-10-16 11:39:47 +00:00
|
|
|
ok, err := s.metaBase.Reload(c.metaOpts...)
|
|
|
|
if err != nil {
|
|
|
|
if errors.Is(err, meta.ErrDegradedMode) {
|
2023-04-12 14:35:10 +00:00
|
|
|
s.log.Error(logs.ShardCantOpenMetabaseMoveToADegradedMode, zap.Error(err))
|
2022-10-16 11:39:47 +00:00
|
|
|
_ = s.setMode(mode.DegradedReadOnly)
|
|
|
|
}
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
if ok {
|
2022-10-16 13:11:29 +00:00
|
|
|
var err error
|
2022-10-16 11:39:47 +00:00
|
|
|
if c.refillMetabase {
|
2022-10-16 13:11:29 +00:00
|
|
|
// Here we refill metabase only if a new instance was opened. This is a feature,
|
|
|
|
// we don't want to hang for some time just because we forgot to change
|
|
|
|
// config after the node was updated.
|
2023-04-12 14:01:29 +00:00
|
|
|
err = s.refillMetabase(ctx)
|
2022-10-16 11:39:47 +00:00
|
|
|
} else {
|
2022-10-16 13:11:29 +00:00
|
|
|
err = s.metaBase.Init()
|
|
|
|
}
|
|
|
|
if err != nil {
|
2023-04-12 14:35:10 +00:00
|
|
|
s.log.Error(logs.ShardCantInitializeMetabaseMoveToADegradedreadonlyMode, zap.Error(err))
|
2022-10-16 13:11:29 +00:00
|
|
|
_ = s.setMode(mode.DegradedReadOnly)
|
|
|
|
return err
|
2022-10-16 11:39:47 +00:00
|
|
|
}
|
|
|
|
}
|
2024-06-18 08:11:56 +00:00
|
|
|
return s.setMode(c.info.Mode)
|
2022-10-16 11:39:47 +00:00
|
|
|
}
|
2023-05-23 12:02:38 +00:00
|
|
|
|
|
|
|
func (s *Shard) lockExclusive() func() {
|
|
|
|
s.setModeRequested.Store(true)
|
|
|
|
val := s.gcCancel.Load()
|
|
|
|
if val != nil {
|
|
|
|
cancelGC := val.(context.CancelFunc)
|
|
|
|
cancelGC()
|
|
|
|
}
|
2024-07-31 13:30:07 +00:00
|
|
|
if c := s.writecacheSealCancel.Load(); c != nil {
|
|
|
|
c.cancel()
|
|
|
|
}
|
2023-05-23 12:02:38 +00:00
|
|
|
s.m.Lock()
|
|
|
|
s.setModeRequested.Store(false)
|
|
|
|
return s.m.Unlock
|
|
|
|
}
|