[#746] morph: Implement and use multi-client

There is a need to work with a set of Neo RPC nodes in order not to depend
on the failure of some nodes while others are active.

Support "multi-client" mode of morph `Client` entity. If instance is not
"multi-client", it works as before. Constructor `New` creates multi-client,
and each method performs iterating over the fixed set of endpoints until
success. Opened client connections are cached (without eviction for now).

Storage (as earlier) and IR (from now) nodes can be configured with multiple
Neo endpoints. As above, `New` creates multi-client instance, so we don't
need initialization changes on app-side.

`Wait` and `GetDesignateHash` methods of `Client` return an error from now
to detect connection errors. `NotaryEnabled` method is removed as unused.

Signed-off-by: Leonard Lyubich <leonard@nspcc.ru>

pkg/morph/client/multi.go

@@ -0,0 +1,105 @@
+package client
+
+import (
+	"sync"
+
+	"github.com/nspcc-dev/neo-go/pkg/core/native/nativenames"
+	"github.com/nspcc-dev/neo-go/pkg/rpc/client"
+	"github.com/nspcc-dev/neo-go/pkg/wallet"
+	"go.uber.org/zap"
+)
+
+type multiClient struct {
+	cfg cfg
+
+	account *wallet.Account
+
+	sharedNotary *notary // notary config needed for single client construction
+
+	endpoints  []string
+	clientsMtx sync.Mutex
+	clients    map[string]*Client
+}
+
+// createForAddress creates single Client instance using provided endpoint.
+//
+// note: must be wrapped into mutex lock.
+func (x *multiClient) createForAddress(addr string) (*Client, error) {
+	cli, err := client.New(x.cfg.ctx, addr, client.Options{
+		DialTimeout: x.cfg.dialTimeout,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	err = cli.Init() // magic number is set there based on RPC node answer
+	if err != nil {
+		return nil, err
+	}
+
+	gas, err := cli.GetNativeContractHash(nativenames.Gas)
+	if err != nil {
+		return nil, err
+	}
+
+	c := &Client{
+		singleClient: &singleClient{
+			logger:       x.cfg.logger,
+			client:       cli,
+			acc:          x.account,
+			gas:          gas,
+			waitInterval: x.cfg.waitInterval,
+			signer:       x.cfg.signer,
+			notary:       x.sharedNotary,
+		},
+	}
+
+	x.clients[addr] = c
+
+	return c, nil
+}
+
+func (x *multiClient) iterateClients(f func(*Client) error) error {
+	var (
+		firstErr error
+		err      error
+	)
+
+	for i := range x.endpoints {
+		select {
+		case <-x.cfg.ctx.Done():
+			return x.cfg.ctx.Err()
+		default:
+		}
+
+		x.clientsMtx.Lock()
+
+		c, cached := x.clients[x.endpoints[i]]
+		if !cached {
+			c, err = x.createForAddress(x.endpoints[i])
+		}
+
+		x.clientsMtx.Unlock()
+
+		if !cached && err != nil {
+			x.cfg.logger.Error("could not open morph client connection",
+				zap.String("endpoint", x.endpoints[i]),
+				zap.String("err", err.Error()),
+			)
+		} else {
+			err = f(c)
+		}
+
+		success := err == nil
+
+		if success || firstErr == nil {
+			firstErr = err
+
+			if success {
+				break
+			}
+		}
+	}
+
+	return firstErr
+}