From b4cb54e7ed83c1595129aa29e89db2d003707567 Mon Sep 17 00:00:00 2001
From: Airat Arifullin <a.arifullin@yadro.com>
Date: Thu, 29 Feb 2024 18:31:04 +0300
Subject: [PATCH] [#963] node: Go on initialization even deposit notary is hung

* Make makeAndWaitNotaryDeposit run asynchronously as worker
  during application boot-up.

Signed-off-by: Airat Arifullin <a.arifullin@yadro.com>
---
 cmd/frostfs-node/config.go |  6 ++++++
 cmd/frostfs-node/main.go   | 16 ++++++++++++++--
 cmd/frostfs-node/morph.go  |  2 ++
 internal/logs/logs.go      |  2 ++
 4 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go
index 48b76ac0f..40fa34a95 100644
--- a/cmd/frostfs-node/config.go
+++ b/cmd/frostfs-node/config.go
@@ -352,6 +352,12 @@ type internals struct {
 
 	log *logger.Logger
 
+	// Some methods that initialize node components may launch an asynchronous job,
+	// appending it to workers. That means despite application is already running,
+	// not all components completely initialized yet.
+	// initAppWG is used to control this situation.
+	initAppWG sync.WaitGroup
+
 	wg      sync.WaitGroup
 	workers []worker
 	closers []closer
diff --git a/cmd/frostfs-node/main.go b/cmd/frostfs-node/main.go
index e1eeb6861..f82b2f171 100644
--- a/cmd/frostfs-node/main.go
+++ b/cmd/frostfs-node/main.go
@@ -67,7 +67,10 @@ func main() {
 
 	bootUp(ctx, c)
 
-	c.compareAndSwapHealthStatus(control.HealthStatus_STARTING, control.HealthStatus_READY)
+	go func() {
+		c.initAppWG.Wait()
+		c.compareAndSwapHealthStatus(control.HealthStatus_STARTING, control.HealthStatus_READY)
+	}()
 
 	wait(c)
 }
@@ -144,7 +147,16 @@ func stopAndLog(c *cfg, name string, stopper func() error) {
 func bootUp(ctx context.Context, c *cfg) {
 	runAndLog(ctx, c, "NATS", true, connectNats)
 	runAndLog(ctx, c, "gRPC", false, func(_ context.Context, c *cfg) { serveGRPC(c) })
-	runAndLog(ctx, c, "notary", true, makeAndWaitNotaryDeposit)
+
+	// It may happen that boot-up waits for the execution of a notary deposit transaction
+	// and waiting loop may hang for an indefinite time. In this case, we need to let
+	// frostfs-node go on initialization, although its functionality will be available partially.
+	// That's why makeAndWaitNotaryDeposit is run asynchroniosly.
+	c.initAppWG.Add(1)
+	c.workers = append(c.workers, newWorkerFromFunc(func(ctx context.Context) {
+		defer c.initAppWG.Done()
+		runAndLog(ctx, c, "notary", true, makeAndWaitNotaryDeposit)
+	}))
 
 	bootstrapNode(c)
 	startWorkers(ctx, c)
diff --git a/cmd/frostfs-node/morph.go b/cmd/frostfs-node/morph.go
index 698fb3b83..1b148095b 100644
--- a/cmd/frostfs-node/morph.go
+++ b/cmd/frostfs-node/morph.go
@@ -158,6 +158,7 @@ var (
 
 func waitNotaryDeposit(ctx context.Context, c *cfg, tx util.Uint256) error {
 	for i := 0; i < notaryDepositRetriesAmount; i++ {
+		c.log.Debug(logs.ClientAttemptToWaitForNotaryDepositTransactionToGetPersisted)
 		select {
 		case <-ctx.Done():
 			return ctx.Err()
@@ -167,6 +168,7 @@ func waitNotaryDeposit(ctx context.Context, c *cfg, tx util.Uint256) error {
 		ok, err := c.cfgMorph.client.TxHalt(tx)
 		if err == nil {
 			if ok {
+				c.log.Info(logs.ClientNotaryDepositTransactionWasSuccessfullyPersisted)
 				return nil
 			}
 
diff --git a/internal/logs/logs.go b/internal/logs/logs.go
index 6b48940b7..bd8b8ed01 100644
--- a/internal/logs/logs.go
+++ b/internal/logs/logs.go
@@ -158,6 +158,8 @@ const (
 	ClientNotaryDepositInvoke                                               = "notary deposit invoke"
 	ClientNotaryRequestWithPreparedMainTXInvoked                            = "notary request with prepared main TX invoked"
 	ClientNotaryRequestInvoked                                              = "notary request invoked"
+	ClientNotaryDepositTransactionWasSuccessfullyPersisted                  = "notary deposit transaction was successfully persisted"
+	ClientAttemptToWaitForNotaryDepositTransactionToGetPersisted            = "attempt to wait for notary deposit transaction to get persisted"
 	ClientNeoClientInvoke                                                   = "neo client invoke"
 	ClientNativeGasTransferInvoke                                           = "native gas transfer invoke"
 	ClientBatchGasTransferInvoke                                            = "batch gas transfer invoke"