acid-ant/frostfs-node
1
0
Fork 0
forked from TrueCloudLab/frostfs-node
Code Issues Pull requests Projects Releases Packages Wiki Activity

[#963] node: Go on initialization even deposit notary is hung

Browse source 
* Make makeAndWaitNotaryDeposit run asynchronously as worker
  during application boot-up.

Signed-off-by: Airat Arifullin <a.arifullin@yadro.com>
This commit is contained in:
Airat Arifullin 2024-02-29 18:31:04 +03:00 committed by Evgenii Stratonikov
parent 6eb63cf5c7
commit b4cb54e7ed
4 changed files with 24 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
cmd/frostfs-node/config.go
View file

@ -352,6 +352,12 @@ type internals struct {
log *logger.Logger
// Some methods that initialize node components may launch an asynchronous job,
// appending it to workers. That means despite application is already running,
// not all components completely initialized yet.
// initAppWG is used to control this situation.
initAppWG sync.WaitGroup
wg sync.WaitGroup
workers []worker
closers []closer

16
cmd/frostfs-node/main.go
View file

@ -67,7 +67,10 @@ func main() {
bootUp(ctx, c)
c.compareAndSwapHealthStatus(control.HealthStatus_STARTING, control.HealthStatus_READY)
go func() {
c.initAppWG.Wait()
c.compareAndSwapHealthStatus(control.HealthStatus_STARTING, control.HealthStatus_READY)
}()
wait(c)
}
@ -144,7 +147,16 @@ func stopAndLog(c *cfg, name string, stopper func() error) {
func bootUp(ctx context.Context, c *cfg) {
runAndLog(ctx, c, "NATS", true, connectNats)
runAndLog(ctx, c, "gRPC", false, func(_ context.Context, c *cfg) { serveGRPC(c) })
runAndLog(ctx, c, "notary", true, makeAndWaitNotaryDeposit)
// It may happen that boot-up waits for the execution of a notary deposit transaction
// and waiting loop may hang for an indefinite time. In this case, we need to let
// frostfs-node go on initialization, although its functionality will be available partially.
// That's why makeAndWaitNotaryDeposit is run asynchroniosly.
c.initAppWG.Add(1)
c.workers = append(c.workers, newWorkerFromFunc(func(ctx context.Context) {
defer c.initAppWG.Done()
runAndLog(ctx, c, "notary", true, makeAndWaitNotaryDeposit)
}))
bootstrapNode(c)
startWorkers(ctx, c)

2
cmd/frostfs-node/morph.go
View file

@ -158,6 +158,7 @@ var (
func waitNotaryDeposit(ctx context.Context, c *cfg, tx util.Uint256) error {
for i := 0; i < notaryDepositRetriesAmount; i++ {
c.log.Debug(logs.ClientAttemptToWaitForNotaryDepositTransactionToGetPersisted)
select {
case <-ctx.Done():
return ctx.Err()
@ -167,6 +168,7 @@ func waitNotaryDeposit(ctx context.Context, c *cfg, tx util.Uint256) error {
ok, err := c.cfgMorph.client.TxHalt(tx)
if err == nil {
if ok {
c.log.Info(logs.ClientNotaryDepositTransactionWasSuccessfullyPersisted)
return nil
}

2
internal/logs/logs.go
View file

@ -158,6 +158,8 @@ const (
ClientNotaryDepositInvoke = "notary deposit invoke"
ClientNotaryRequestWithPreparedMainTXInvoked = "notary request with prepared main TX invoked"
ClientNotaryRequestInvoked = "notary request invoked"
ClientNotaryDepositTransactionWasSuccessfullyPersisted = "notary deposit transaction was successfully persisted"
ClientAttemptToWaitForNotaryDepositTransactionToGetPersisted = "attempt to wait for notary deposit transaction to get persisted"
ClientNeoClientInvoke = "neo client invoke"
ClientNativeGasTransferInvoke = "native gas transfer invoke"
ClientBatchGasTransferInvoke = "batch gas transfer invoke"