c4fc5cb54a
* plugin/pkg/up: make default intervals shorter I think 15 min is too high, make this lower to react faster. Signed-off-by: Miek Gieben <miek@miek.nl> * Update README Signed-off-by: Miek Gieben <miek@miek.nl>
96 lines
2.4 KiB
Go
96 lines
2.4 KiB
Go
// Package up is used to run a function for some duration. If a new function is added while a previous run is
|
|
// still ongoing, nothing new will be executed.
|
|
package up
|
|
|
|
import (
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/cenkalti/backoff/v4"
|
|
)
|
|
|
|
// Probe is used to run a single Func until it returns true (indicating a target is healthy). If an Func
|
|
// is already in progress no new one will be added, i.e. there is always a maximum of 1 checks in flight.
|
|
// When failures start to happen we will back off every second failure up to maximum of 4 intervals.
|
|
type Probe struct {
|
|
sync.Mutex
|
|
inprogress int
|
|
expBackoff backoff.ExponentialBackOff
|
|
}
|
|
|
|
// Func is used to determine if a target is alive. If so this function must return nil.
|
|
type Func func() error
|
|
|
|
// New returns a pointer to an initialized Probe.
|
|
func New() *Probe { return &Probe{} }
|
|
|
|
// Do will probe target, if a probe is already in progress this is a noop.
|
|
func (p *Probe) Do(f Func) {
|
|
p.Lock()
|
|
if p.inprogress != idle {
|
|
p.Unlock()
|
|
return
|
|
}
|
|
p.inprogress = active
|
|
interval := p.expBackoff.NextBackOff()
|
|
// If exponential backoff has reached the maximum elapsed time (15 minutes),
|
|
// reset it and try again
|
|
if interval == -1 {
|
|
p.expBackoff.Reset()
|
|
interval = p.expBackoff.NextBackOff()
|
|
}
|
|
p.Unlock()
|
|
// Passed the lock. Now run f for as long it returns false. If a true is returned
|
|
// we return from the goroutine and we can accept another Func to run.
|
|
go func() {
|
|
i := 1
|
|
for {
|
|
if err := f(); err == nil {
|
|
break
|
|
}
|
|
time.Sleep(interval)
|
|
p.Lock()
|
|
if p.inprogress == stop {
|
|
p.Unlock()
|
|
return
|
|
}
|
|
p.Unlock()
|
|
i++
|
|
}
|
|
|
|
p.Lock()
|
|
p.inprogress = idle
|
|
p.Unlock()
|
|
}()
|
|
}
|
|
|
|
// Stop stops the probing.
|
|
func (p *Probe) Stop() {
|
|
p.Lock()
|
|
p.inprogress = stop
|
|
p.Unlock()
|
|
}
|
|
|
|
// Start will initialize the probe manager, after which probes can be initiated with Do.
|
|
// Initializes exponential backoff using the given interval duration
|
|
func (p *Probe) Start(interval time.Duration) {
|
|
p.Lock()
|
|
eB := &backoff.ExponentialBackOff{
|
|
InitialInterval: interval,
|
|
RandomizationFactor: backoff.DefaultRandomizationFactor,
|
|
Multiplier: backoff.DefaultMultiplier,
|
|
MaxInterval: 15 * time.Second,
|
|
MaxElapsedTime: 2 * time.Minute,
|
|
Stop: backoff.Stop,
|
|
Clock: backoff.SystemClock,
|
|
}
|
|
p.expBackoff = *eB
|
|
p.expBackoff.Reset()
|
|
p.Unlock()
|
|
}
|
|
|
|
const (
|
|
idle = iota
|
|
active
|
|
stop
|
|
)
|