[#963] node: Go on initialization even deposit notary is hung #1014

Merged
fyrchik merged 1 commit from aarifullin/frostfs-node:fix/963-control_svc_iface_down into master 2024-03-12 10:18:18 +00:00
4 changed files with 24 additions and 2 deletions

View file

@ -352,6 +352,12 @@ type internals struct {
log *logger.Logger log *logger.Logger
// Some methods that initialize node components may launch an asynchronous job,
// appending it to workers. That means despite application is already running,
// not all components completely initialized yet.
// initAppWG is used to control this situation.
initAppWG sync.WaitGroup
wg sync.WaitGroup wg sync.WaitGroup
workers []worker workers []worker
closers []closer closers []closer

View file

@ -67,7 +67,10 @@ func main() {
bootUp(ctx, c) bootUp(ctx, c)
go func() {
c.initAppWG.Wait()
c.compareAndSwapHealthStatus(control.HealthStatus_STARTING, control.HealthStatus_READY) c.compareAndSwapHealthStatus(control.HealthStatus_STARTING, control.HealthStatus_READY)
}()
wait(c) wait(c)
} }
@ -144,7 +147,16 @@ func stopAndLog(c *cfg, name string, stopper func() error) {
func bootUp(ctx context.Context, c *cfg) { func bootUp(ctx context.Context, c *cfg) {
runAndLog(ctx, c, "NATS", true, connectNats) runAndLog(ctx, c, "NATS", true, connectNats)
runAndLog(ctx, c, "gRPC", false, func(_ context.Context, c *cfg) { serveGRPC(c) }) runAndLog(ctx, c, "gRPC", false, func(_ context.Context, c *cfg) { serveGRPC(c) })
// It may happen that boot-up waits for the execution of a notary deposit transaction
// and waiting loop may hang for an indefinite time. In this case, we need to let
// frostfs-node go on initialization, although its functionality will be available partially.
// That's why makeAndWaitNotaryDeposit is run asynchroniosly.
c.initAppWG.Add(1)
c.workers = append(c.workers, newWorkerFromFunc(func(ctx context.Context) {
defer c.initAppWG.Done()
runAndLog(ctx, c, "notary", true, makeAndWaitNotaryDeposit) runAndLog(ctx, c, "notary", true, makeAndWaitNotaryDeposit)
}))
bootstrapNode(c) bootstrapNode(c)
startWorkers(ctx, c) startWorkers(ctx, c)

View file

@ -158,6 +158,7 @@ var (
func waitNotaryDeposit(ctx context.Context, c *cfg, tx util.Uint256) error { func waitNotaryDeposit(ctx context.Context, c *cfg, tx util.Uint256) error {
for i := 0; i < notaryDepositRetriesAmount; i++ { for i := 0; i < notaryDepositRetriesAmount; i++ {
c.log.Debug(logs.ClientAttemptToWaitForNotaryDepositTransactionToGetPersisted)

ClientAttemptToWaitForNotaryDepositTransactionGetsPersisted -> ClientAttemptToWaitForNotaryDepositTransactionToGetPersisted

ClientAttemptToWaitForNotaryDepositTransactionGetsPersisted -> ClientAttemptToWaitForNotaryDepositTransactionToGetPersisted
select { select {
case <-ctx.Done(): case <-ctx.Done():
return ctx.Err() return ctx.Err()
@ -167,6 +168,7 @@ func waitNotaryDeposit(ctx context.Context, c *cfg, tx util.Uint256) error {
ok, err := c.cfgMorph.client.TxHalt(tx) ok, err := c.cfgMorph.client.TxHalt(tx)
if err == nil { if err == nil {
if ok { if ok {
c.log.Info(logs.ClientNotaryDepositTransactionWasSuccessfullyPersisted)
return nil return nil
} }

View file

@ -158,6 +158,8 @@ const (
ClientNotaryDepositInvoke = "notary deposit invoke" ClientNotaryDepositInvoke = "notary deposit invoke"
ClientNotaryRequestWithPreparedMainTXInvoked = "notary request with prepared main TX invoked" ClientNotaryRequestWithPreparedMainTXInvoked = "notary request with prepared main TX invoked"
ClientNotaryRequestInvoked = "notary request invoked" ClientNotaryRequestInvoked = "notary request invoked"
ClientNotaryDepositTransactionWasSuccessfullyPersisted = "notary deposit transaction was successfully persisted"
ClientAttemptToWaitForNotaryDepositTransactionToGetPersisted = "attempt to wait for notary deposit transaction to get persisted"

It is either for notary deposit transaction to get persisted or until notary deposit transaction is persisted

It is either `for notary deposit transaction to get persisted` or `until notary deposit transaction is persisted`

Fixed: I preferred the first one

Fixed: I preferred the first one
ClientNeoClientInvoke = "neo client invoke" ClientNeoClientInvoke = "neo client invoke"
ClientNativeGasTransferInvoke = "native gas transfer invoke" ClientNativeGasTransferInvoke = "native gas transfer invoke"
ClientBatchGasTransferInvoke = "batch gas transfer invoke" ClientBatchGasTransferInvoke = "batch gas transfer invoke"