[#206] Make caches' options configurable

Now caches' options can be configured in .yanl file.
Export caches' variables (listobjects and objects).

Signed-off-by: Angira Kekteeva <kira@nspcc.ru>
This commit is contained in:
Angira Kekteeva 2021-08-16 12:58:09 +03:00 committed by Kirillov Denis
parent 376642565f
commit b1c6629b10
6 changed files with 77 additions and 8 deletions

View file

@ -144,6 +144,17 @@ If parameter doesn't support environment variable (e.g. `--listen_address 0.0.0.
listen_address: 0.0.0.0:8084
```
#### Cache parameters
Parameters for caches in s3-gw can be specified in .yaml config file. E.g.:
```
cache:
lifetime: 300s
size: 150
list_objects_lifetime: 1m
```
If invalid values are set, the gateway will use default values instead.
## NeoFS AuthMate
Authmate is a tool to create gateway AWS credentials. AWS users

View file

@ -15,9 +15,9 @@ type ObjectsCache interface {
}
const (
// DefaultObjectsCacheLifetime is a default lifetime of objects in cache.
// DefaultObjectsCacheLifetime is a default lifetime of entries in objects' cache.
DefaultObjectsCacheLifetime = time.Minute * 5
// DefaultObjectsCacheSize is a default maximum number of objects in cache.
// DefaultObjectsCacheSize is a default maximum number of entries in objects' in cache.
DefaultObjectsCacheSize = 1e6
)

View file

@ -30,6 +30,13 @@ type (
objCache cache.ObjectsCache
}
// CacheConfig contains params for caches.
CacheConfig struct {
Lifetime time.Duration
Size int
ListObjectsLifetime time.Duration
}
// Params stores basic API parameters.
Params struct {
Pool pool.Pool
@ -130,12 +137,12 @@ const (
// NewLayer creates instance of layer. It checks credentials
// and establishes gRPC connection with node.
func NewLayer(log *zap.Logger, conns pool.Pool) Client {
func NewLayer(log *zap.Logger, conns pool.Pool, config *CacheConfig) Client {
return &layer{
pool: conns,
log: log,
listObjCache: newListObjectsCache(defaultObjectsListCacheLifetime),
objCache: cache.New(cache.DefaultObjectsCacheSize, cache.DefaultObjectsCacheLifetime),
listObjCache: newListObjectsCache(config.ListObjectsLifetime),
objCache: cache.New(config.Size, config.Lifetime),
}
}

View file

@ -13,7 +13,7 @@ import (
request.
The cache is a map which has a key: cacheOptions struct and a value: list of objects. After putting a record we
start a timer (via time.AfterFunc) that removes the record after defaultObjectsListCacheLifetime value.
start a timer (via time.AfterFunc) that removes the record after DefaultObjectsListCacheLifetime value.
When we get a request from the user we just try to find the suitable and non-expired cache and then we return
the list of objects. Otherwise we send the request to NeoFS.
@ -27,7 +27,8 @@ type (
}
)
const defaultObjectsListCacheLifetime = time.Second * 60
// DefaultObjectsListCacheLifetime is a default lifetime of entries in cache of ListObjects.
const DefaultObjectsListCacheLifetime = time.Second * 60
type (
listObjectsCache struct {

View file

@ -54,6 +54,11 @@ const ( // Settings.
cfgRequestTimeout = "request_timeout"
cfgRebalanceTimer = "rebalance_timer"
// Caching.
cfgObjectsCacheLifetime = "cache.lifetime"
cfgCacheSize = "cache.size"
cfgListObjectsCacheLifetime = "cache.list_objects_lifetime"
// MaxClients.
cfgMaxClientsCount = "max_clients_count"
cfgMaxClientsDeadline = "max_clients_deadline"

View file

@ -10,6 +10,7 @@ import (
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/auth"
"github.com/nspcc-dev/neofs-s3-gw/api/cache"
"github.com/nspcc-dev/neofs-s3-gw/api/handler"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
"github.com/nspcc-dev/neofs-s3-gw/internal/wallet"
@ -108,8 +109,10 @@ func newApp(ctx context.Context, l *zap.Logger, v *viper.Viper) *App {
l.Fatal("failed to create connection pool", zap.Error(err))
}
cacheCfg := getCacheOptions(v, l)
// prepare object layer
obj = layer.NewLayer(l, conns)
obj = layer.NewLayer(l, conns, cacheCfg)
// prepare auth center
ctr = auth.New(conns, key)
@ -207,3 +210,45 @@ func (a *App) Server(ctx context.Context) {
close(a.webDone)
}
func getCacheOptions(v *viper.Viper, l *zap.Logger) *layer.CacheConfig {
cacheCfg := layer.CacheConfig{
ListObjectsLifetime: layer.DefaultObjectsListCacheLifetime,
Size: cache.DefaultObjectsCacheSize,
Lifetime: cache.DefaultObjectsCacheLifetime,
}
if v.IsSet(cfgObjectsCacheLifetime) {
lifetime := v.GetDuration(cfgObjectsCacheLifetime)
if lifetime <= 0 {
l.Error("invalid cache lifetime, using default value (in seconds)",
zap.Duration("value in config", lifetime),
zap.Duration("default", cacheCfg.Lifetime))
} else {
cacheCfg.Lifetime = lifetime
}
}
if v.IsSet(cfgCacheSize) {
size := v.GetInt(cfgCacheSize)
if size <= 0 {
l.Error("invalid cache size, using default value",
zap.Int("value in config", size),
zap.Int("default", cacheCfg.Size))
} else {
cacheCfg.Size = size
}
}
if v.IsSet(cfgListObjectsCacheLifetime) {
lifetime := v.GetDuration(cfgListObjectsCacheLifetime)
if lifetime <= 0 {
l.Error("invalid list objects cache lifetime, using default value (in seconds)",
zap.Duration("value in config", lifetime),
zap.Duration("default", cacheCfg.ListObjectsLifetime))
} else {
cacheCfg.ListObjectsLifetime = lifetime
}
}
return &cacheCfg
}