Prepare to export

This commit is contained in:
Evgeniy Kulikov 2020-07-13 02:00:47 +03:00
parent 0bdd3e4507
commit 151c3c672a
8 changed files with 138 additions and 21 deletions

View file

@ -5,15 +5,18 @@ import (
"net/http" "net/http"
"github.com/gorilla/mux" "github.com/gorilla/mux"
"go.uber.org/atomic"
) )
type Healthy interface {
Status() error
}
const ( const (
healthyState = "NeoFS S3 Gateway is " healthyState = "NeoFS S3 Gateway is "
// defaultContentType = "text/plain; charset=utf-8" // defaultContentType = "text/plain; charset=utf-8"
) )
func attachHealthy(r *mux.Router, e *atomic.Error) { func attachHealthy(r *mux.Router, h Healthy) {
r.HandleFunc("/-/ready", func(w http.ResponseWriter, r *http.Request) { r.HandleFunc("/-/ready", func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK) w.WriteHeader(http.StatusOK)
_, _ = fmt.Fprintln(w, healthyState+"ready") _, _ = fmt.Fprintln(w, healthyState+"ready")
@ -23,7 +26,7 @@ func attachHealthy(r *mux.Router, e *atomic.Error) {
code := http.StatusOK code := http.StatusOK
msg := "healthy" msg := "healthy"
if err := e.Load(); err != nil { if err := h.Status(); err != nil {
msg = "unhealthy: " + err.Error() msg = "unhealthy: " + err.Error()
code = http.StatusBadRequest code = http.StatusBadRequest
} }

View file

@ -4,12 +4,14 @@ import (
"github.com/gorilla/mux" "github.com/gorilla/mux"
"github.com/prometheus/client_golang/prometheus/promhttp" "github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/spf13/viper" "github.com/spf13/viper"
"go.uber.org/zap"
) )
func attachMetrics(v *viper.Viper, r *mux.Router) { func attachMetrics(v *viper.Viper, l *zap.Logger, r *mux.Router) {
if !v.GetBool(cfgEnableMetrics) { if !v.GetBool(cfgEnableMetrics) {
return return
} }
l.Info("enable metrics")
r.Handle("/metrics", promhttp.Handler()) r.Handle("/metrics", promhttp.Handler())
} }

View file

@ -5,13 +5,16 @@ import (
"github.com/gorilla/mux" "github.com/gorilla/mux"
"github.com/spf13/viper" "github.com/spf13/viper"
"go.uber.org/zap"
) )
func attachProfiler(v *viper.Viper, r *mux.Router) { func attachProfiler(v *viper.Viper, l *zap.Logger, r *mux.Router) {
if !v.GetBool(cfgEnableProfiler) { if !v.GetBool(cfgEnableProfiler) {
return return
} }
l.Info("enable profiler")
r.HandleFunc("/debug/pprof/", pprof.Index) r.HandleFunc("/debug/pprof/", pprof.Index)
r.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) r.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
r.HandleFunc("/debug/pprof/profile", pprof.Profile) r.HandleFunc("/debug/pprof/profile", pprof.Profile)

View file

@ -32,9 +32,10 @@ const (
defaultTTL = minimumTTLInMinutes * time.Minute defaultTTL = minimumTTLInMinutes * time.Minute
defaultRebalanceTimer = 15 * time.Second defaultRebalanceTimer = 15 * time.Second
defaultRequestTimeout = 15 * time.Second defaultRequestTimeout = 15 * time.Second
defaultConnectTimeout = 30 * time.Second defaultConnectTimeout = 30 * time.Second
defaultShutdownTimeout = 15 * time.Second
defaultKeepaliveTime = 10 * time.Second defaultKeepaliveTime = 10 * time.Second
defaultKeepaliveTimeout = 10 * time.Second defaultKeepaliveTimeout = 10 * time.Second

View file

@ -2,8 +2,11 @@ package main
import ( import (
"context" "context"
"net"
"net/http"
"time" "time"
"github.com/gorilla/mux"
minio "github.com/minio/minio/legacy" minio "github.com/minio/minio/legacy"
"github.com/minio/minio/neofs/layer" "github.com/minio/minio/neofs/layer"
"github.com/minio/minio/neofs/pool" "github.com/minio/minio/neofs/pool"
@ -19,12 +22,15 @@ type (
App struct { App struct {
cli pool.Pool cli pool.Pool
log *zap.Logger log *zap.Logger
web *mux.Router
cfg *viper.Viper cfg *viper.Viper
obj minio.ObjectLayer obj minio.ObjectLayer
conTimeout time.Duration conTimeout time.Duration
reqTimeout time.Duration reqTimeout time.Duration
reBalance time.Duration
webDone chan struct{} webDone chan struct{}
wrkDone chan struct{} wrkDone chan struct{}
} }
@ -40,6 +46,8 @@ func newApp(l *zap.Logger, v *viper.Viper) *App {
key = fetchKey(l, v) key = fetchKey(l, v)
reBalance = defaultRebalanceTimer
conTimeout = defaultConnectTimeout conTimeout = defaultConnectTimeout
reqTimeout = defaultRequestTimeout reqTimeout = defaultRequestTimeout
) )
@ -68,6 +76,10 @@ func newApp(l *zap.Logger, v *viper.Viper) *App {
ClientParameters: keepalive.ClientParameters{}, ClientParameters: keepalive.ClientParameters{},
} }
if v := v.GetDuration(cfgRebalanceTimer); v > 0 {
reBalance = v
}
if cli, err = pool.New(poolConfig); err != nil { if cli, err = pool.New(poolConfig); err != nil {
l.Fatal("could not prepare pool connections", l.Fatal("could not prepare pool connections",
zap.Error(err)) zap.Error(err))
@ -108,38 +120,90 @@ func newApp(l *zap.Logger, v *viper.Viper) *App {
cli: cli, cli: cli,
log: l, log: l,
cfg: v, cfg: v,
web: minio.NewRouter(obj),
webDone: make(chan struct{}, 1), webDone: make(chan struct{}, 1),
wrkDone: make(chan struct{}, 1), wrkDone: make(chan struct{}, 1),
reBalance: reBalance,
conTimeout: conTimeout, conTimeout: conTimeout,
reqTimeout: reqTimeout, reqTimeout: reqTimeout,
} }
} }
func (a *App) Wait(ctx context.Context) { func (a *App) Wait() {
defer a.log.Info("application finished")
a.log.Info("application started") a.log.Info("application started")
select { select {
case <-a.wrkDone: // wait for worker is stopped case <-a.wrkDone: // wait for worker is stopped
<-a.webDone <-a.webDone
case <-a.webDone: // wait for web-server is stopped case <-a.webDone: // wait for web-server is stopped
<-a.wrkDone <-a.wrkDone
} }
a.log.Info("application finished")
} }
func (a *App) Server(ctx context.Context) { func (a *App) Server(ctx context.Context) {
defer func() { var (
<-ctx.Done() err error
a.log.Info("stopping server") lis net.Listener
close(a.webDone) lic net.ListenConfig
srv = http.Server{Handler: a.web}
addr = a.cfg.GetString(cfgListenAddress)
)
if lis, err = lic.Listen(ctx, "tcp", addr); err != nil {
a.log.Fatal("could not prepare listener",
zap.Error(err))
}
// Attach app-specific routes:
attachHealthy(a.web, a.cli)
attachMetrics(a.cfg, a.log, a.web)
attachProfiler(a.cfg, a.log, a.web)
go func() {
a.log.Info("starting server",
zap.String("bind", addr))
if err = srv.Serve(lis); err != nil && err != http.ErrServerClosed {
a.log.Warn("listen and serve",
zap.Error(err))
}
}() }()
<-ctx.Done()
ctx, cancel := context.WithTimeout(context.Background(), defaultShutdownTimeout)
defer cancel()
a.log.Info("stopping server",
zap.Error(srv.Shutdown(ctx)))
close(a.webDone)
} }
func (a *App) Worker(ctx context.Context) { func (a *App) Worker(ctx context.Context) {
defer func() { tick := time.NewTimer(a.reBalance)
<-ctx.Done()
a.log.Info("stopping worker") loop:
close(a.wrkDone) for {
}() select {
case <-ctx.Done():
break loop
case <-tick.C:
ctx, cancel := context.WithTimeout(ctx, a.conTimeout)
a.cli.ReBalance(ctx)
cancel()
tick.Reset(a.reBalance)
}
}
tick.Stop()
a.cli.Close()
a.log.Info("stopping worker")
close(a.wrkDone)
} }

View file

@ -11,5 +11,5 @@ func main() {
go a.Server(g) go a.Server(g)
go a.Worker(g) go a.Worker(g)
a.Wait(g) a.Wait()
} }

44
legacy/neofs-router.go Normal file
View file

@ -0,0 +1,44 @@
package legacy
import (
"github.com/gorilla/mux"
)
func NewRouter(obj ObjectLayer) *mux.Router {
// Initialize router. `SkipClean(true)` stops gorilla/mux from
// normalizing URL path minio/minio#3256
// avoid URL path encoding minio/minio#8950
router := mux.NewRouter().SkipClean(true).UseEncodedPath()
// Add healthcheck router
registerHealthCheckRouter(router)
// Add server metrics router
registerMetricsRouter(router)
// Add API router.
registerAPIRouter(router, true, true)
layer := NewGatewayLayerWithLocker(obj)
// Once endpoints are finalized, initialize the new object api in safe mode.
globalObjLayerMutex.Lock()
globalSafeMode = true
globalObjectAPI = layer
globalObjLayerMutex.Unlock()
// Calls all New() for all sub-systems.
newAllSubsystems()
// Verify if object layer supports
// - encryption
// - compression
verifyObjectLayerFeatures("gateway NeoFS", layer)
// Disable safe mode operation, after all initialization is over.
globalObjLayerMutex.Lock()
globalSafeMode = false
globalObjLayerMutex.Unlock()
return router
}

View file

@ -182,7 +182,7 @@ func (p *pool) ReBalance(ctx context.Context) {
keys := make(map[uint32]struct{}) keys := make(map[uint32]struct{})
p.log.Info("re-balancing connections") p.log.Debug("re-balancing connections")
for i := range p.nodes { for i := range p.nodes {
var ( var (