forked from TrueCloudLab/frostfs-s3-gw
[#206] Make caches' options configurable
Now caches' options can be configured in .yanl file. Export caches' variables (listobjects and objects). Signed-off-by: Angira Kekteeva <kira@nspcc.ru>
This commit is contained in:
parent
376642565f
commit
b1c6629b10
6 changed files with 77 additions and 8 deletions
11
README.md
11
README.md
|
@ -144,6 +144,17 @@ If parameter doesn't support environment variable (e.g. `--listen_address 0.0.0.
|
||||||
listen_address: 0.0.0.0:8084
|
listen_address: 0.0.0.0:8084
|
||||||
```
|
```
|
||||||
|
|
||||||
|
#### Cache parameters
|
||||||
|
|
||||||
|
Parameters for caches in s3-gw can be specified in .yaml config file. E.g.:
|
||||||
|
```
|
||||||
|
cache:
|
||||||
|
lifetime: 300s
|
||||||
|
size: 150
|
||||||
|
list_objects_lifetime: 1m
|
||||||
|
```
|
||||||
|
If invalid values are set, the gateway will use default values instead.
|
||||||
|
|
||||||
## NeoFS AuthMate
|
## NeoFS AuthMate
|
||||||
|
|
||||||
Authmate is a tool to create gateway AWS credentials. AWS users
|
Authmate is a tool to create gateway AWS credentials. AWS users
|
||||||
|
|
4
api/cache/object_cache.go
vendored
4
api/cache/object_cache.go
vendored
|
@ -15,9 +15,9 @@ type ObjectsCache interface {
|
||||||
}
|
}
|
||||||
|
|
||||||
const (
|
const (
|
||||||
// DefaultObjectsCacheLifetime is a default lifetime of objects in cache.
|
// DefaultObjectsCacheLifetime is a default lifetime of entries in objects' cache.
|
||||||
DefaultObjectsCacheLifetime = time.Minute * 5
|
DefaultObjectsCacheLifetime = time.Minute * 5
|
||||||
// DefaultObjectsCacheSize is a default maximum number of objects in cache.
|
// DefaultObjectsCacheSize is a default maximum number of entries in objects' in cache.
|
||||||
DefaultObjectsCacheSize = 1e6
|
DefaultObjectsCacheSize = 1e6
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -30,6 +30,13 @@ type (
|
||||||
objCache cache.ObjectsCache
|
objCache cache.ObjectsCache
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// CacheConfig contains params for caches.
|
||||||
|
CacheConfig struct {
|
||||||
|
Lifetime time.Duration
|
||||||
|
Size int
|
||||||
|
ListObjectsLifetime time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
// Params stores basic API parameters.
|
// Params stores basic API parameters.
|
||||||
Params struct {
|
Params struct {
|
||||||
Pool pool.Pool
|
Pool pool.Pool
|
||||||
|
@ -130,12 +137,12 @@ const (
|
||||||
|
|
||||||
// NewLayer creates instance of layer. It checks credentials
|
// NewLayer creates instance of layer. It checks credentials
|
||||||
// and establishes gRPC connection with node.
|
// and establishes gRPC connection with node.
|
||||||
func NewLayer(log *zap.Logger, conns pool.Pool) Client {
|
func NewLayer(log *zap.Logger, conns pool.Pool, config *CacheConfig) Client {
|
||||||
return &layer{
|
return &layer{
|
||||||
pool: conns,
|
pool: conns,
|
||||||
log: log,
|
log: log,
|
||||||
listObjCache: newListObjectsCache(defaultObjectsListCacheLifetime),
|
listObjCache: newListObjectsCache(config.ListObjectsLifetime),
|
||||||
objCache: cache.New(cache.DefaultObjectsCacheSize, cache.DefaultObjectsCacheLifetime),
|
objCache: cache.New(config.Size, config.Lifetime),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -13,7 +13,7 @@ import (
|
||||||
request.
|
request.
|
||||||
|
|
||||||
The cache is a map which has a key: cacheOptions struct and a value: list of objects. After putting a record we
|
The cache is a map which has a key: cacheOptions struct and a value: list of objects. After putting a record we
|
||||||
start a timer (via time.AfterFunc) that removes the record after defaultObjectsListCacheLifetime value.
|
start a timer (via time.AfterFunc) that removes the record after DefaultObjectsListCacheLifetime value.
|
||||||
|
|
||||||
When we get a request from the user we just try to find the suitable and non-expired cache and then we return
|
When we get a request from the user we just try to find the suitable and non-expired cache and then we return
|
||||||
the list of objects. Otherwise we send the request to NeoFS.
|
the list of objects. Otherwise we send the request to NeoFS.
|
||||||
|
@ -27,7 +27,8 @@ type (
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
const defaultObjectsListCacheLifetime = time.Second * 60
|
// DefaultObjectsListCacheLifetime is a default lifetime of entries in cache of ListObjects.
|
||||||
|
const DefaultObjectsListCacheLifetime = time.Second * 60
|
||||||
|
|
||||||
type (
|
type (
|
||||||
listObjectsCache struct {
|
listObjectsCache struct {
|
||||||
|
|
|
@ -54,6 +54,11 @@ const ( // Settings.
|
||||||
cfgRequestTimeout = "request_timeout"
|
cfgRequestTimeout = "request_timeout"
|
||||||
cfgRebalanceTimer = "rebalance_timer"
|
cfgRebalanceTimer = "rebalance_timer"
|
||||||
|
|
||||||
|
// Caching.
|
||||||
|
cfgObjectsCacheLifetime = "cache.lifetime"
|
||||||
|
cfgCacheSize = "cache.size"
|
||||||
|
cfgListObjectsCacheLifetime = "cache.list_objects_lifetime"
|
||||||
|
|
||||||
// MaxClients.
|
// MaxClients.
|
||||||
cfgMaxClientsCount = "max_clients_count"
|
cfgMaxClientsCount = "max_clients_count"
|
||||||
cfgMaxClientsDeadline = "max_clients_deadline"
|
cfgMaxClientsDeadline = "max_clients_deadline"
|
||||||
|
|
|
@ -10,6 +10,7 @@ import (
|
||||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api"
|
"github.com/nspcc-dev/neofs-s3-gw/api"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/auth"
|
"github.com/nspcc-dev/neofs-s3-gw/api/auth"
|
||||||
|
"github.com/nspcc-dev/neofs-s3-gw/api/cache"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/handler"
|
"github.com/nspcc-dev/neofs-s3-gw/api/handler"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
|
||||||
"github.com/nspcc-dev/neofs-s3-gw/internal/wallet"
|
"github.com/nspcc-dev/neofs-s3-gw/internal/wallet"
|
||||||
|
@ -108,8 +109,10 @@ func newApp(ctx context.Context, l *zap.Logger, v *viper.Viper) *App {
|
||||||
l.Fatal("failed to create connection pool", zap.Error(err))
|
l.Fatal("failed to create connection pool", zap.Error(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
cacheCfg := getCacheOptions(v, l)
|
||||||
|
|
||||||
// prepare object layer
|
// prepare object layer
|
||||||
obj = layer.NewLayer(l, conns)
|
obj = layer.NewLayer(l, conns, cacheCfg)
|
||||||
|
|
||||||
// prepare auth center
|
// prepare auth center
|
||||||
ctr = auth.New(conns, key)
|
ctr = auth.New(conns, key)
|
||||||
|
@ -207,3 +210,45 @@ func (a *App) Server(ctx context.Context) {
|
||||||
|
|
||||||
close(a.webDone)
|
close(a.webDone)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func getCacheOptions(v *viper.Viper, l *zap.Logger) *layer.CacheConfig {
|
||||||
|
cacheCfg := layer.CacheConfig{
|
||||||
|
ListObjectsLifetime: layer.DefaultObjectsListCacheLifetime,
|
||||||
|
Size: cache.DefaultObjectsCacheSize,
|
||||||
|
Lifetime: cache.DefaultObjectsCacheLifetime,
|
||||||
|
}
|
||||||
|
|
||||||
|
if v.IsSet(cfgObjectsCacheLifetime) {
|
||||||
|
lifetime := v.GetDuration(cfgObjectsCacheLifetime)
|
||||||
|
if lifetime <= 0 {
|
||||||
|
l.Error("invalid cache lifetime, using default value (in seconds)",
|
||||||
|
zap.Duration("value in config", lifetime),
|
||||||
|
zap.Duration("default", cacheCfg.Lifetime))
|
||||||
|
} else {
|
||||||
|
cacheCfg.Lifetime = lifetime
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if v.IsSet(cfgCacheSize) {
|
||||||
|
size := v.GetInt(cfgCacheSize)
|
||||||
|
if size <= 0 {
|
||||||
|
l.Error("invalid cache size, using default value",
|
||||||
|
zap.Int("value in config", size),
|
||||||
|
zap.Int("default", cacheCfg.Size))
|
||||||
|
} else {
|
||||||
|
cacheCfg.Size = size
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if v.IsSet(cfgListObjectsCacheLifetime) {
|
||||||
|
lifetime := v.GetDuration(cfgListObjectsCacheLifetime)
|
||||||
|
if lifetime <= 0 {
|
||||||
|
l.Error("invalid list objects cache lifetime, using default value (in seconds)",
|
||||||
|
zap.Duration("value in config", lifetime),
|
||||||
|
zap.Duration("default", cacheCfg.ListObjectsLifetime))
|
||||||
|
} else {
|
||||||
|
cacheCfg.ListObjectsLifetime = lifetime
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return &cacheCfg
|
||||||
|
}
|
||||||
|
|
Loading…
Reference in a new issue