[#206] Make caches' options configurable

Now caches' options can be configured in .yanl file.
Export caches' variables (listobjects and objects).

Signed-off-by: Angira Kekteeva <kira@nspcc.ru>
This commit is contained in:
Angira Kekteeva 2021-08-16 12:58:09 +03:00 committed by Kirillov Denis
parent 376642565f
commit b1c6629b10
6 changed files with 77 additions and 8 deletions

View file

@ -144,6 +144,17 @@ If parameter doesn't support environment variable (e.g. `--listen_address 0.0.0.
listen_address: 0.0.0.0:8084 listen_address: 0.0.0.0:8084
``` ```
#### Cache parameters
Parameters for caches in s3-gw can be specified in .yaml config file. E.g.:
```
cache:
lifetime: 300s
size: 150
list_objects_lifetime: 1m
```
If invalid values are set, the gateway will use default values instead.
## NeoFS AuthMate ## NeoFS AuthMate
Authmate is a tool to create gateway AWS credentials. AWS users Authmate is a tool to create gateway AWS credentials. AWS users

View file

@ -15,9 +15,9 @@ type ObjectsCache interface {
} }
const ( const (
// DefaultObjectsCacheLifetime is a default lifetime of objects in cache. // DefaultObjectsCacheLifetime is a default lifetime of entries in objects' cache.
DefaultObjectsCacheLifetime = time.Minute * 5 DefaultObjectsCacheLifetime = time.Minute * 5
// DefaultObjectsCacheSize is a default maximum number of objects in cache. // DefaultObjectsCacheSize is a default maximum number of entries in objects' in cache.
DefaultObjectsCacheSize = 1e6 DefaultObjectsCacheSize = 1e6
) )

View file

@ -30,6 +30,13 @@ type (
objCache cache.ObjectsCache objCache cache.ObjectsCache
} }
// CacheConfig contains params for caches.
CacheConfig struct {
Lifetime time.Duration
Size int
ListObjectsLifetime time.Duration
}
// Params stores basic API parameters. // Params stores basic API parameters.
Params struct { Params struct {
Pool pool.Pool Pool pool.Pool
@ -130,12 +137,12 @@ const (
// NewLayer creates instance of layer. It checks credentials // NewLayer creates instance of layer. It checks credentials
// and establishes gRPC connection with node. // and establishes gRPC connection with node.
func NewLayer(log *zap.Logger, conns pool.Pool) Client { func NewLayer(log *zap.Logger, conns pool.Pool, config *CacheConfig) Client {
return &layer{ return &layer{
pool: conns, pool: conns,
log: log, log: log,
listObjCache: newListObjectsCache(defaultObjectsListCacheLifetime), listObjCache: newListObjectsCache(config.ListObjectsLifetime),
objCache: cache.New(cache.DefaultObjectsCacheSize, cache.DefaultObjectsCacheLifetime), objCache: cache.New(config.Size, config.Lifetime),
} }
} }

View file

@ -13,7 +13,7 @@ import (
request. request.
The cache is a map which has a key: cacheOptions struct and a value: list of objects. After putting a record we The cache is a map which has a key: cacheOptions struct and a value: list of objects. After putting a record we
start a timer (via time.AfterFunc) that removes the record after defaultObjectsListCacheLifetime value. start a timer (via time.AfterFunc) that removes the record after DefaultObjectsListCacheLifetime value.
When we get a request from the user we just try to find the suitable and non-expired cache and then we return When we get a request from the user we just try to find the suitable and non-expired cache and then we return
the list of objects. Otherwise we send the request to NeoFS. the list of objects. Otherwise we send the request to NeoFS.
@ -27,7 +27,8 @@ type (
} }
) )
const defaultObjectsListCacheLifetime = time.Second * 60 // DefaultObjectsListCacheLifetime is a default lifetime of entries in cache of ListObjects.
const DefaultObjectsListCacheLifetime = time.Second * 60
type ( type (
listObjectsCache struct { listObjectsCache struct {

View file

@ -54,6 +54,11 @@ const ( // Settings.
cfgRequestTimeout = "request_timeout" cfgRequestTimeout = "request_timeout"
cfgRebalanceTimer = "rebalance_timer" cfgRebalanceTimer = "rebalance_timer"
// Caching.
cfgObjectsCacheLifetime = "cache.lifetime"
cfgCacheSize = "cache.size"
cfgListObjectsCacheLifetime = "cache.list_objects_lifetime"
// MaxClients. // MaxClients.
cfgMaxClientsCount = "max_clients_count" cfgMaxClientsCount = "max_clients_count"
cfgMaxClientsDeadline = "max_clients_deadline" cfgMaxClientsDeadline = "max_clients_deadline"

View file

@ -10,6 +10,7 @@ import (
"github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/crypto/keys"
"github.com/nspcc-dev/neofs-s3-gw/api" "github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/auth" "github.com/nspcc-dev/neofs-s3-gw/api/auth"
"github.com/nspcc-dev/neofs-s3-gw/api/cache"
"github.com/nspcc-dev/neofs-s3-gw/api/handler" "github.com/nspcc-dev/neofs-s3-gw/api/handler"
"github.com/nspcc-dev/neofs-s3-gw/api/layer" "github.com/nspcc-dev/neofs-s3-gw/api/layer"
"github.com/nspcc-dev/neofs-s3-gw/internal/wallet" "github.com/nspcc-dev/neofs-s3-gw/internal/wallet"
@ -108,8 +109,10 @@ func newApp(ctx context.Context, l *zap.Logger, v *viper.Viper) *App {
l.Fatal("failed to create connection pool", zap.Error(err)) l.Fatal("failed to create connection pool", zap.Error(err))
} }
cacheCfg := getCacheOptions(v, l)
// prepare object layer // prepare object layer
obj = layer.NewLayer(l, conns) obj = layer.NewLayer(l, conns, cacheCfg)
// prepare auth center // prepare auth center
ctr = auth.New(conns, key) ctr = auth.New(conns, key)
@ -207,3 +210,45 @@ func (a *App) Server(ctx context.Context) {
close(a.webDone) close(a.webDone)
} }
func getCacheOptions(v *viper.Viper, l *zap.Logger) *layer.CacheConfig {
cacheCfg := layer.CacheConfig{
ListObjectsLifetime: layer.DefaultObjectsListCacheLifetime,
Size: cache.DefaultObjectsCacheSize,
Lifetime: cache.DefaultObjectsCacheLifetime,
}
if v.IsSet(cfgObjectsCacheLifetime) {
lifetime := v.GetDuration(cfgObjectsCacheLifetime)
if lifetime <= 0 {
l.Error("invalid cache lifetime, using default value (in seconds)",
zap.Duration("value in config", lifetime),
zap.Duration("default", cacheCfg.Lifetime))
} else {
cacheCfg.Lifetime = lifetime
}
}
if v.IsSet(cfgCacheSize) {
size := v.GetInt(cfgCacheSize)
if size <= 0 {
l.Error("invalid cache size, using default value",
zap.Int("value in config", size),
zap.Int("default", cacheCfg.Size))
} else {
cacheCfg.Size = size
}
}
if v.IsSet(cfgListObjectsCacheLifetime) {
lifetime := v.GetDuration(cfgListObjectsCacheLifetime)
if lifetime <= 0 {
l.Error("invalid list objects cache lifetime, using default value (in seconds)",
zap.Duration("value in config", lifetime),
zap.Duration("default", cacheCfg.ListObjectsLifetime))
} else {
cacheCfg.ListObjectsLifetime = lifetime
}
}
return &cacheCfg
}