oracle: reprocess request on fail

pkg/services/oracle/oracle.go

@@ -40,6 +40,8 @@ type (
 		// respMtx protects responses map.
 		respMtx   sync.RWMutex
 		responses map[uint64]*incompleteTx
+		// removed contains ids of requests which won't be processed further due to expiration.
+		removed map[uint64]bool
 
 		wallet *wallet.Wallet
 	}
@@ -82,6 +84,12 @@ type (
 const (
 	// defaultRequestTimeout is default request timeout.
 	defaultRequestTimeout = time.Second * 5
+
+	// defaultMaxTaskTimeout is default timeout for the request to be dropped if it can't be processed.
+	defaultMaxTaskTimeout = time.Hour
+
+	// defaultRefreshInterval is default timeout for the failed request to be reprocessed.
+	defaultRefreshInterval = time.Minute * 3
 )
 
 // NewOracle returns new oracle instance.
@@ -92,6 +100,7 @@ func NewOracle(cfg Config) (*Oracle, error) {
 		close:      make(chan struct{}),
 		requestMap: make(chan map[uint64]*state.OracleRequest, 1),
 		responses:  make(map[uint64]*incompleteTx),
+		removed:    make(map[uint64]bool),
 	}
 	if o.MainCfg.RequestTimeout == 0 {
 		o.MainCfg.RequestTimeout = defaultRequestTimeout
@@ -100,6 +109,12 @@ func NewOracle(cfg Config) (*Oracle, error) {
 		o.MainCfg.MaxConcurrentRequests = defaultMaxConcurrentRequests
 	}
 	o.requestCh = make(chan request, o.MainCfg.MaxConcurrentRequests)
+	if o.MainCfg.MaxTaskTimeout == 0 {
+		o.MainCfg.MaxTaskTimeout = defaultMaxTaskTimeout
+	}
+	if o.MainCfg.RefreshInterval == 0 {
+		o.MainCfg.RefreshInterval = defaultRefreshInterval
+	}
 
 	var err error
 	w := cfg.MainCfg.UnlockWallet
@@ -147,10 +162,35 @@ func (o *Oracle) Run() {
 	for i := 0; i < o.MainCfg.MaxConcurrentRequests; i++ {
 		go o.runRequestWorker()
 	}
+
+	tick := time.NewTicker(o.MainCfg.RefreshInterval)
 	for {
 		select {
 		case <-o.close:
+			tick.Stop()
 			return
+		case <-tick.C:
+			var reprocess []uint64
+			o.respMtx.RLock()
+			o.removed = make(map[uint64]bool)
+			for id, incTx := range o.responses {
+				incTx.RLock()
+				since := time.Since(incTx.time)
+				if since > o.MainCfg.MaxTaskTimeout {
+					o.removed[id] = true
+				} else if since > o.MainCfg.RefreshInterval {
+					reprocess = append(reprocess, id)
+				}
+				incTx.RUnlock()
+			}
+			for id := range o.removed {
+				delete(o.responses, id)
+			}
+			o.respMtx.Unlock()
+
+			for _, id := range reprocess {
+				o.requestCh <- request{ID: id}
+			}
 		case reqs := <-o.requestMap:
 			for id, req := range reqs {
 				o.requestCh <- request{