[#963] node: Go on initialization even deposit notary is hung #1014
4 changed files with 24 additions and 2 deletions
|
@ -352,6 +352,12 @@ type internals struct {
|
||||||
|
|
||||||
log *logger.Logger
|
log *logger.Logger
|
||||||
|
|
||||||
|
// Some methods that initialize node components may launch an asynchronous job,
|
||||||
|
// appending it to workers. That means despite application is already running,
|
||||||
|
// not all components completely initialized yet.
|
||||||
|
// initAppWG is used to control this situation.
|
||||||
|
initAppWG sync.WaitGroup
|
||||||
|
|
||||||
wg sync.WaitGroup
|
wg sync.WaitGroup
|
||||||
workers []worker
|
workers []worker
|
||||||
closers []closer
|
closers []closer
|
||||||
|
|
|
@ -67,7 +67,10 @@ func main() {
|
||||||
|
|
||||||
bootUp(ctx, c)
|
bootUp(ctx, c)
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
c.initAppWG.Wait()
|
||||||
c.compareAndSwapHealthStatus(control.HealthStatus_STARTING, control.HealthStatus_READY)
|
c.compareAndSwapHealthStatus(control.HealthStatus_STARTING, control.HealthStatus_READY)
|
||||||
|
}()
|
||||||
|
|
||||||
wait(c)
|
wait(c)
|
||||||
}
|
}
|
||||||
|
@ -144,7 +147,16 @@ func stopAndLog(c *cfg, name string, stopper func() error) {
|
||||||
func bootUp(ctx context.Context, c *cfg) {
|
func bootUp(ctx context.Context, c *cfg) {
|
||||||
runAndLog(ctx, c, "NATS", true, connectNats)
|
runAndLog(ctx, c, "NATS", true, connectNats)
|
||||||
runAndLog(ctx, c, "gRPC", false, func(_ context.Context, c *cfg) { serveGRPC(c) })
|
runAndLog(ctx, c, "gRPC", false, func(_ context.Context, c *cfg) { serveGRPC(c) })
|
||||||
|
|
||||||
|
// It may happen that boot-up waits for the execution of a notary deposit transaction
|
||||||
|
// and waiting loop may hang for an indefinite time. In this case, we need to let
|
||||||
|
// frostfs-node go on initialization, although its functionality will be available partially.
|
||||||
|
// That's why makeAndWaitNotaryDeposit is run asynchroniosly.
|
||||||
|
c.initAppWG.Add(1)
|
||||||
|
c.workers = append(c.workers, newWorkerFromFunc(func(ctx context.Context) {
|
||||||
|
defer c.initAppWG.Done()
|
||||||
runAndLog(ctx, c, "notary", true, makeAndWaitNotaryDeposit)
|
runAndLog(ctx, c, "notary", true, makeAndWaitNotaryDeposit)
|
||||||
|
}))
|
||||||
|
|
||||||
bootstrapNode(c)
|
bootstrapNode(c)
|
||||||
startWorkers(ctx, c)
|
startWorkers(ctx, c)
|
||||||
|
|
|
@ -158,6 +158,7 @@ var (
|
||||||
|
|
||||||
func waitNotaryDeposit(ctx context.Context, c *cfg, tx util.Uint256) error {
|
func waitNotaryDeposit(ctx context.Context, c *cfg, tx util.Uint256) error {
|
||||||
for i := 0; i < notaryDepositRetriesAmount; i++ {
|
for i := 0; i < notaryDepositRetriesAmount; i++ {
|
||||||
|
c.log.Debug(logs.ClientAttemptToWaitForNotaryDepositTransactionToGetPersisted)
|
||||||
|
|||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
return ctx.Err()
|
return ctx.Err()
|
||||||
|
@ -167,6 +168,7 @@ func waitNotaryDeposit(ctx context.Context, c *cfg, tx util.Uint256) error {
|
||||||
ok, err := c.cfgMorph.client.TxHalt(tx)
|
ok, err := c.cfgMorph.client.TxHalt(tx)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
if ok {
|
if ok {
|
||||||
|
c.log.Info(logs.ClientNotaryDepositTransactionWasSuccessfullyPersisted)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -158,6 +158,8 @@ const (
|
||||||
ClientNotaryDepositInvoke = "notary deposit invoke"
|
ClientNotaryDepositInvoke = "notary deposit invoke"
|
||||||
ClientNotaryRequestWithPreparedMainTXInvoked = "notary request with prepared main TX invoked"
|
ClientNotaryRequestWithPreparedMainTXInvoked = "notary request with prepared main TX invoked"
|
||||||
ClientNotaryRequestInvoked = "notary request invoked"
|
ClientNotaryRequestInvoked = "notary request invoked"
|
||||||
|
ClientNotaryDepositTransactionWasSuccessfullyPersisted = "notary deposit transaction was successfully persisted"
|
||||||
|
ClientAttemptToWaitForNotaryDepositTransactionToGetPersisted = "attempt to wait for notary deposit transaction to get persisted"
|
||||||
fyrchik
commented
It is either It is either `for notary deposit transaction to get persisted` or `until notary deposit transaction is persisted`
aarifullin
commented
Fixed: I preferred the first one Fixed: I preferred the first one
|
|||||||
ClientNeoClientInvoke = "neo client invoke"
|
ClientNeoClientInvoke = "neo client invoke"
|
||||||
ClientNativeGasTransferInvoke = "native gas transfer invoke"
|
ClientNativeGasTransferInvoke = "native gas transfer invoke"
|
||||||
ClientBatchGasTransferInvoke = "batch gas transfer invoke"
|
ClientBatchGasTransferInvoke = "batch gas transfer invoke"
|
||||||
|
|
Loading…
Reference in a new issue
ClientAttemptToWaitForNotaryDepositTransactionGetsPersisted -> ClientAttemptToWaitForNotaryDepositTransactionToGetPersisted