[#963] node: Go on initialization even deposit notary is hung

* Make makeAndWaitNotaryDeposit run asynchronously as worker
  during application boot-up.

Signed-off-by: Airat Arifullin <a.arifullin@yadro.com>

cmd/frostfs-node/config.go

@@ -352,6 +352,12 @@ type internals struct {
 
 	log *logger.Logger
 
+	// Some methods that initialize node components may launch an asynchronous job,
+	// appending it to workers. That means despite application is already running,
+	// not all components completely initialized yet.
+	// initAppWG is used to control this situation.
+	initAppWG sync.WaitGroup
+
 	wg      sync.WaitGroup
 	workers []worker
 	closers []closer

cmd/frostfs-node/main.go

@@ -67,7 +67,10 @@ func main() {
 
 	bootUp(ctx, c)
 
-	c.compareAndSwapHealthStatus(control.HealthStatus_STARTING, control.HealthStatus_READY)
+	go func() {
+		c.initAppWG.Wait()
+		c.compareAndSwapHealthStatus(control.HealthStatus_STARTING, control.HealthStatus_READY)
+	}()
 
 	wait(c)
 }
@@ -144,7 +147,16 @@ func stopAndLog(c *cfg, name string, stopper func() error) {
 func bootUp(ctx context.Context, c *cfg) {
 	runAndLog(ctx, c, "NATS", true, connectNats)
 	runAndLog(ctx, c, "gRPC", false, func(_ context.Context, c *cfg) { serveGRPC(c) })
-	runAndLog(ctx, c, "notary", true, makeAndWaitNotaryDeposit)
+
+	// It may happen that boot-up waits for the execution of a notary deposit transaction
+	// and waiting loop may hang for an indefinite time. In this case, we need to let
+	// frostfs-node go on initialization, although its functionality will be available partially.
+	// That's why makeAndWaitNotaryDeposit is run asynchroniosly.
+	c.initAppWG.Add(1)
+	c.workers = append(c.workers, newWorkerFromFunc(func(ctx context.Context) {
+		defer c.initAppWG.Done()
+		runAndLog(ctx, c, "notary", true, makeAndWaitNotaryDeposit)
+	}))
 
 	bootstrapNode(c)
 	startWorkers(ctx, c)

cmd/frostfs-node/morph.go

@@ -158,6 +158,7 @@ var (
 
 func waitNotaryDeposit(ctx context.Context, c *cfg, tx util.Uint256) error {
 	for i := 0; i < notaryDepositRetriesAmount; i++ {
+		c.log.Debug(logs.ClientAttemptToWaitForNotaryDepositTransactionToGetPersisted)
 		select {
 		case <-ctx.Done():
 			return ctx.Err()
@@ -167,6 +168,7 @@ func waitNotaryDeposit(ctx context.Context, c *cfg, tx util.Uint256) error {
 		ok, err := c.cfgMorph.client.TxHalt(tx)
 		if err == nil {
 			if ok {
+				c.log.Info(logs.ClientNotaryDepositTransactionWasSuccessfullyPersisted)
 				return nil
 			}
 

internal/logs/logs.go

@@ -158,6 +158,8 @@ const (
 	ClientNotaryDepositInvoke                                               = "notary deposit invoke"
 	ClientNotaryRequestWithPreparedMainTXInvoked                            = "notary request with prepared main TX invoked"
 	ClientNotaryRequestInvoked                                              = "notary request invoked"
+	ClientNotaryDepositTransactionWasSuccessfullyPersisted                  = "notary deposit transaction was successfully persisted"
+	ClientAttemptToWaitForNotaryDepositTransactionToGetPersisted            = "attempt to wait for notary deposit transaction to get persisted"
 	ClientNeoClientInvoke                                                   = "neo client invoke"
 	ClientNativeGasTransferInvoke                                           = "native gas transfer invoke"
 	ClientBatchGasTransferInvoke                                            = "batch gas transfer invoke"