From 95d847d611e4188e6f4a1ec080f3426ef15c8ef8 Mon Sep 17 00:00:00 2001 From: Marina Biryukova Date: Thu, 12 Dec 2024 13:20:26 +0300 Subject: [PATCH] [#577] Update SDK to support new tree/pool version Signed-off-by: Marina Biryukova --- api/cache/buckets.go | 55 ++++++++++++++++++++--- api/cache/cache_test.go | 2 +- api/cache/network.go | 86 ++++++++++++++++++++++++++++++++++++ api/cache/network_info.go | 65 --------------------------- api/data/info.go | 2 + api/handler/handlers_test.go | 11 ++++- api/layer/cache.go | 34 +++++++++++--- api/layer/container.go | 2 + api/layer/frostfs/frostfs.go | 3 ++ api/layer/frostfs_mock.go | 4 ++ api/layer/versioning_test.go | 11 ++++- cmd/s3-gw/app.go | 67 +++++++++++++++------------- cmd/s3-gw/app_settings.go | 7 +-- config/config.env | 4 +- config/config.yaml | 4 +- docs/configuration.md | 12 ++--- go.mod | 13 +++++- go.sum | 37 +++++++++++++++- internal/frostfs/frostfs.go | 9 ++++ internal/frostfs/source.go | 66 +++++++++++++++++++++++++++ internal/logs/logs.go | 1 + 21 files changed, 371 insertions(+), 124 deletions(-) create mode 100644 api/cache/network.go delete mode 100644 api/cache/network_info.go create mode 100644 internal/frostfs/source.go diff --git a/api/cache/buckets.go b/api/cache/buckets.go index 5271b8e..5efc776 100644 --- a/api/cache/buckets.go +++ b/api/cache/buckets.go @@ -6,14 +6,16 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "github.com/bluele/gcache" "go.uber.org/zap" ) // BucketCache contains cache with objects and the lifetime of cache entries. type BucketCache struct { - cache gcache.Cache - logger *zap.Logger + cache gcache.Cache + cidCache gcache.Cache + logger *zap.Logger } const ( @@ -33,14 +35,45 @@ func DefaultBucketConfig(logger *zap.Logger) *Config { } // NewBucketCache creates an object of BucketCache. -func NewBucketCache(config *Config) *BucketCache { - gc := gcache.New(config.Size).LRU().Expiration(config.Lifetime).Build() - return &BucketCache{cache: gc, logger: config.Logger} +func NewBucketCache(config *Config, cidCache bool) *BucketCache { + cache := &BucketCache{ + cache: gcache.New(config.Size).LRU().Expiration(config.Lifetime).Build(), + logger: config.Logger, + } + + if cidCache { + cache.cidCache = gcache.New(config.Size).LRU().Expiration(config.Lifetime).Build() + } + return cache } // Get returns a cached object. func (o *BucketCache) Get(ns, bktName string) *data.BucketInfo { - entry, err := o.cache.Get(formKey(ns, bktName)) + return o.get(formKey(ns, bktName)) +} + +func (o *BucketCache) GetByCID(cnrID cid.ID) *data.BucketInfo { + if o.cidCache == nil { + return nil + } + + entry, err := o.cidCache.Get(cnrID) + if err != nil { + return nil + } + + key, ok := entry.(string) + if !ok { + o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)), + zap.String("expected", fmt.Sprintf("%T", key))) + return nil + } + + return o.get(key) +} + +func (o *BucketCache) get(key string) *data.BucketInfo { + entry, err := o.cache.Get(key) if err != nil { return nil } @@ -57,11 +90,21 @@ func (o *BucketCache) Get(ns, bktName string) *data.BucketInfo { // Put puts an object to cache. func (o *BucketCache) Put(bkt *data.BucketInfo) error { + if o.cidCache != nil { + if err := o.cidCache.Set(bkt.CID, formKey(bkt.Zone, bkt.Name)); err != nil { + return err + } + } + return o.cache.Set(formKey(bkt.Zone, bkt.Name), bkt) } // Delete deletes an object from cache. func (o *BucketCache) Delete(bkt *data.BucketInfo) bool { + if o.cidCache != nil { + o.cidCache.Remove(bkt.CID) + } + return o.cache.Remove(formKey(bkt.Zone, bkt.Name)) } diff --git a/api/cache/cache_test.go b/api/cache/cache_test.go index 61bb64c..36230d9 100644 --- a/api/cache/cache_test.go +++ b/api/cache/cache_test.go @@ -42,7 +42,7 @@ func TestAccessBoxCacheType(t *testing.T) { func TestBucketsCacheType(t *testing.T) { logger, observedLog := getObservedLogger() - cache := NewBucketCache(DefaultBucketConfig(logger)) + cache := NewBucketCache(DefaultBucketConfig(logger), false) bktInfo := &data.BucketInfo{Name: "bucket"} diff --git a/api/cache/network.go b/api/cache/network.go new file mode 100644 index 0000000..899055e --- /dev/null +++ b/api/cache/network.go @@ -0,0 +1,86 @@ +package cache + +import ( + "fmt" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" + "github.com/bluele/gcache" + "go.uber.org/zap" +) + +type ( + // NetworkCache provides cache for network-related values. + NetworkCache struct { + cache gcache.Cache + logger *zap.Logger + } + + // NetworkCacheConfig stores expiration params for cache. + NetworkCacheConfig struct { + Lifetime time.Duration + Logger *zap.Logger + } +) + +const ( + DefaultNetworkCacheLifetime = 1 * time.Minute + networkCacheSize = 2 + networkInfoKey = "network_info" + netmapKey = "netmap" +) + +// DefaultNetworkConfig returns new default cache expiration values. +func DefaultNetworkConfig(logger *zap.Logger) *NetworkCacheConfig { + return &NetworkCacheConfig{ + Lifetime: DefaultNetworkCacheLifetime, + Logger: logger, + } +} + +// NewNetworkCache creates an object of NetworkCache. +func NewNetworkCache(config *NetworkCacheConfig) *NetworkCache { + gc := gcache.New(networkCacheSize).LRU().Expiration(config.Lifetime).Build() + return &NetworkCache{cache: gc, logger: config.Logger} +} + +func (c *NetworkCache) GetNetworkInfo() *netmap.NetworkInfo { + entry, err := c.cache.Get(networkInfoKey) + if err != nil { + return nil + } + + result, ok := entry.(netmap.NetworkInfo) + if !ok { + c.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)), + zap.String("expected", fmt.Sprintf("%T", result))) + return nil + } + + return &result +} + +func (c *NetworkCache) PutNetworkInfo(info netmap.NetworkInfo) error { + return c.cache.Set(networkInfoKey, info) +} + +func (c *NetworkCache) GetNetmap() *netmap.NetMap { + entry, err := c.cache.Get(netmapKey) + if err != nil { + return nil + } + + result, ok := entry.(netmap.NetMap) + if !ok { + c.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)), + zap.String("expected", fmt.Sprintf("%T", result))) + return nil + } + + return &result +} + +func (c *NetworkCache) PutNetmap(nm netmap.NetMap) error { + return c.cache.Set(netmapKey, nm) +} diff --git a/api/cache/network_info.go b/api/cache/network_info.go deleted file mode 100644 index 92f1a99..0000000 --- a/api/cache/network_info.go +++ /dev/null @@ -1,65 +0,0 @@ -package cache - -import ( - "fmt" - "time" - - "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" - "github.com/bluele/gcache" - "go.uber.org/zap" -) - -type ( - // NetworkInfoCache provides cache for network info. - NetworkInfoCache struct { - cache gcache.Cache - logger *zap.Logger - } - - // NetworkInfoCacheConfig stores expiration params for cache. - NetworkInfoCacheConfig struct { - Lifetime time.Duration - Logger *zap.Logger - } -) - -const ( - DefaultNetworkInfoCacheLifetime = 1 * time.Minute - networkInfoCacheSize = 1 - networkInfoKey = "network_info" -) - -// DefaultNetworkInfoConfig returns new default cache expiration values. -func DefaultNetworkInfoConfig(logger *zap.Logger) *NetworkInfoCacheConfig { - return &NetworkInfoCacheConfig{ - Lifetime: DefaultNetworkInfoCacheLifetime, - Logger: logger, - } -} - -// NewNetworkInfoCache creates an object of NetworkInfoCache. -func NewNetworkInfoCache(config *NetworkInfoCacheConfig) *NetworkInfoCache { - gc := gcache.New(networkInfoCacheSize).LRU().Expiration(config.Lifetime).Build() - return &NetworkInfoCache{cache: gc, logger: config.Logger} -} - -func (c *NetworkInfoCache) Get() *netmap.NetworkInfo { - entry, err := c.cache.Get(networkInfoKey) - if err != nil { - return nil - } - - result, ok := entry.(netmap.NetworkInfo) - if !ok { - c.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)), - zap.String("expected", fmt.Sprintf("%T", result))) - return nil - } - - return &result -} - -func (c *NetworkInfoCache) Put(info netmap.NetworkInfo) error { - return c.cache.Set(networkInfoKey, info) -} diff --git a/api/data/info.go b/api/data/info.go index 61b7b71..9dd17cb 100644 --- a/api/data/info.go +++ b/api/data/info.go @@ -6,6 +6,7 @@ import ( "time" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" @@ -32,6 +33,7 @@ type ( LocationConstraint string ObjectLockEnabled bool HomomorphicHashDisabled bool + PlacementPolicy netmap.PlacementPolicy } // ObjectInfo holds S3 object data. diff --git a/api/handler/handlers_test.go b/api/handler/handlers_test.go index 8c886a1..dae87fc 100644 --- a/api/handler/handlers_test.go +++ b/api/handler/handlers_test.go @@ -256,7 +256,7 @@ func getMinCacheConfig(logger *zap.Logger) *layer.CachesConfig { Buckets: minCacheCfg, System: minCacheCfg, AccessControl: minCacheCfg, - NetworkInfo: &cache.NetworkInfoCacheConfig{Lifetime: minCacheCfg.Lifetime}, + Network: &cache.NetworkCacheConfig{Lifetime: minCacheCfg.Lifetime}, } } @@ -404,6 +404,7 @@ func createTestBucketWithLock(hc *handlerContext, bktName string, conf *data.Obj Creator: hc.owner, Name: bktName, AdditionalAttributes: [][2]string{{layer.AttributeLockEnabled, "true"}}, + Policy: getPlacementPolicy(), }) require.NoError(hc.t, err) @@ -415,6 +416,7 @@ func createTestBucketWithLock(hc *handlerContext, bktName string, conf *data.Obj ObjectLockEnabled: true, Owner: ownerID, HomomorphicHashDisabled: res.HomomorphicHashDisabled, + PlacementPolicy: getPlacementPolicy(), } key, err := keys.NewPrivateKey() @@ -534,3 +536,10 @@ func readResponse(t *testing.T, w *httptest.ResponseRecorder, status int, model require.NoError(t, err) } } + +func getPlacementPolicy() (p netmap.PlacementPolicy) { + var r netmap.ReplicaDescriptor + r.SetNumberOfObjects(1) + p.AddReplicas([]netmap.ReplicaDescriptor{r}...) + return p +} diff --git a/api/layer/cache.go b/api/layer/cache.go index c3ceb7c..1ef92a6 100644 --- a/api/layer/cache.go +++ b/api/layer/cache.go @@ -20,7 +20,7 @@ type Cache struct { bucketCache *cache.BucketCache systemCache *cache.SystemCache accessCache *cache.AccessControlCache - networkInfoCache *cache.NetworkInfoCache + networkCache *cache.NetworkCache } // CachesConfig contains params for caches. @@ -33,7 +33,8 @@ type CachesConfig struct { Buckets *cache.Config System *cache.Config AccessControl *cache.Config - NetworkInfo *cache.NetworkInfoCacheConfig + Network *cache.NetworkCacheConfig + CIDCache bool } // DefaultCachesConfigs returns filled configs. @@ -47,7 +48,7 @@ func DefaultCachesConfigs(logger *zap.Logger) *CachesConfig { Buckets: cache.DefaultBucketConfig(logger), System: cache.DefaultSystemConfig(logger), AccessControl: cache.DefaultAccessControlConfig(logger), - NetworkInfo: cache.DefaultNetworkInfoConfig(logger), + Network: cache.DefaultNetworkConfig(logger), } } @@ -58,10 +59,10 @@ func NewCache(cfg *CachesConfig) *Cache { sessionListCache: cache.NewListSessionCache(cfg.SessionList), objCache: cache.New(cfg.Objects), namesCache: cache.NewObjectsNameCache(cfg.Names), - bucketCache: cache.NewBucketCache(cfg.Buckets), + bucketCache: cache.NewBucketCache(cfg.Buckets, cfg.CIDCache), systemCache: cache.NewSystemCache(cfg.System), accessCache: cache.NewAccessControlCache(cfg.AccessControl), - networkInfoCache: cache.NewNetworkInfoCache(cfg.NetworkInfo), + networkCache: cache.NewNetworkCache(cfg.Network), } } @@ -290,11 +291,30 @@ func (c *Cache) DeleteLifecycleConfiguration(bktInfo *data.BucketInfo) { } func (c *Cache) GetNetworkInfo() *netmap.NetworkInfo { - return c.networkInfoCache.Get() + return c.networkCache.GetNetworkInfo() } func (c *Cache) PutNetworkInfo(info netmap.NetworkInfo) { - if err := c.networkInfoCache.Put(info); err != nil { + if err := c.networkCache.PutNetworkInfo(info); err != nil { c.logger.Warn(logs.CouldntCacheNetworkInfo, zap.Error(err)) } } + +func (c *Cache) GetNetmap() *netmap.NetMap { + return c.networkCache.GetNetmap() +} + +func (c *Cache) PutNetmap(nm netmap.NetMap) { + if err := c.networkCache.PutNetmap(nm); err != nil { + c.logger.Warn(logs.CouldntCacheNetmap, zap.Error(err)) + } +} + +func (c *Cache) GetPlacementPolicy(cnrID cid.ID) *netmap.PlacementPolicy { + res := c.bucketCache.GetByCID(cnrID) + if res != nil { + return &res.PlacementPolicy + } + + return nil +} diff --git a/api/layer/container.go b/api/layer/container.go index faa4a60..ebb23d7 100644 --- a/api/layer/container.go +++ b/api/layer/container.go @@ -53,6 +53,7 @@ func (n *Layer) containerInfo(ctx context.Context, prm frostfs.PrmContainer) (*d info.Created = container.CreatedAt(cnr) info.LocationConstraint = cnr.Attribute(attributeLocationConstraint) info.HomomorphicHashDisabled = container.IsHomomorphicHashingDisabled(cnr) + info.PlacementPolicy = cnr.PlacementPolicy() attrLockEnabled := cnr.Attribute(AttributeLockEnabled) if len(attrLockEnabled) > 0 { @@ -148,6 +149,7 @@ func (n *Layer) createContainer(ctx context.Context, p *CreateBucketParams) (*da bktInfo.CID = res.ContainerID bktInfo.HomomorphicHashDisabled = res.HomomorphicHashDisabled + bktInfo.PlacementPolicy = p.Policy n.cache.PutBucket(bktInfo) diff --git a/api/layer/frostfs/frostfs.go b/api/layer/frostfs/frostfs.go index ba01350..9b61240 100644 --- a/api/layer/frostfs/frostfs.go +++ b/api/layer/frostfs/frostfs.go @@ -354,4 +354,7 @@ type FrostFS interface { // Relations returns implementation of relations.Relations interface. Relations() relations.Relations + + // NetmapSnapshot returns information about FrostFS network map. + NetmapSnapshot(context.Context) (netmap.NetMap, error) } diff --git a/api/layer/frostfs_mock.go b/api/layer/frostfs_mock.go index ab983c9..481bbed 100644 --- a/api/layer/frostfs_mock.go +++ b/api/layer/frostfs_mock.go @@ -475,6 +475,10 @@ func (t *TestFrostFS) NetworkInfo(context.Context) (netmap.NetworkInfo, error) { return ni, nil } +func (t *TestFrostFS) NetmapSnapshot(context.Context) (netmap.NetMap, error) { + return netmap.NetMap{}, nil +} + func (t *TestFrostFS) PatchObject(ctx context.Context, prm frostfs.PrmObjectPatch) (oid.ID, error) { obj, err := t.retrieveObject(ctx, prm.Container, prm.Object) if err != nil { diff --git a/api/layer/versioning_test.go b/api/layer/versioning_test.go index 2f49f75..12e1e27 100644 --- a/api/layer/versioning_test.go +++ b/api/layer/versioning_test.go @@ -11,6 +11,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox" bearertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer/test" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" @@ -156,7 +157,8 @@ func prepareContext(t *testing.T, cachesConfig ...*CachesConfig) *testContext { bktName := "testbucket1" res, err := tp.CreateContainer(ctx, frostfs.PrmContainerCreate{ - Name: bktName, + Name: bktName, + Policy: getPlacementPolicy(), }) require.NoError(t, err) @@ -445,3 +447,10 @@ func TestFilterVersionsByMarker(t *testing.T) { }) } } + +func getPlacementPolicy() (p netmap.PlacementPolicy) { + var r netmap.ReplicaDescriptor + r.SetNumberOfObjects(1) + p.AddReplicas([]netmap.ReplicaDescriptor{r}...) + return p +} diff --git a/cmd/s3-gw/app.go b/cmd/s3-gw/app.go index fa9da0a..3645220 100644 --- a/cmd/s3-gw/app.go +++ b/cmd/s3-gw/app.go @@ -71,6 +71,7 @@ type ( key *keys.PrivateKey obj *layer.Layer api api.Handler + cache *layer.Cache frostfsid *frostfsid.FrostFSID @@ -164,15 +165,12 @@ func newApp(ctx context.Context, v *viper.Viper) *App { logSettings := &loggerSettings{} log := pickLogger(v, logSettings) settings := newAppSettings(log, v) - - objPool, treePool, key := getPools(ctx, log.logger, v, settings.dialerSource) + appCache := layer.NewCache(getCacheOptions(v, log.logger)) app := &App{ - log: log.logger, - cfg: v, - pool: objPool, - treePool: treePool, - key: key, + log: log.logger, + cfg: v, + cache: appCache, webDone: make(chan struct{}, 1), wrkDone: make(chan struct{}, 1), @@ -187,6 +185,7 @@ func newApp(ctx context.Context, v *viper.Viper) *App { } func (a *App) init(ctx context.Context) { + a.initPools(ctx) a.initResolver() a.initAuthCenter(ctx) a.setRuntimeParameters() @@ -244,7 +243,7 @@ func (a *App) initLayer(ctx context.Context) { } layerCfg := &layer.Config{ - Cache: layer.NewCache(getCacheOptions(a.cfg, a.log)), + Cache: a.cache, AnonKey: layer.AnonymousKey{ Key: randomKey, }, @@ -757,77 +756,83 @@ func getDialerSource(logger *zap.Logger, cfg *viper.Viper) *internalnet.DialerSo return source } -func getPools(ctx context.Context, logger *zap.Logger, cfg *viper.Viper, dialSource *internalnet.DialerSource) (*pool.Pool, *treepool.Pool, *keys.PrivateKey) { +func (a *App) initPools(ctx context.Context) { var prm pool.InitParameters var prmTree treepool.InitParameters - password := wallet.GetPassword(cfg, cfgWalletPassphrase) - key, err := wallet.GetKeyFromPath(cfg.GetString(cfgWalletPath), cfg.GetString(cfgWalletAddress), password) + password := wallet.GetPassword(a.cfg, cfgWalletPassphrase) + key, err := wallet.GetKeyFromPath(a.cfg.GetString(cfgWalletPath), a.cfg.GetString(cfgWalletAddress), password) if err != nil { - logger.Fatal(logs.CouldNotLoadFrostFSPrivateKey, zap.Error(err)) + a.log.Fatal(logs.CouldNotLoadFrostFSPrivateKey, zap.Error(err)) } prm.SetKey(&key.PrivateKey) prmTree.SetKey(key) - logger.Info(logs.UsingCredentials, zap.String("FrostFS", hex.EncodeToString(key.PublicKey().Bytes()))) + a.log.Info(logs.UsingCredentials, zap.String("FrostFS", hex.EncodeToString(key.PublicKey().Bytes()))) - for _, peer := range fetchPeers(logger, cfg) { + for _, peer := range fetchPeers(a.log, a.cfg) { prm.AddNode(peer) prmTree.AddNode(peer) } - connTimeout := fetchConnectTimeout(cfg) + connTimeout := fetchConnectTimeout(a.cfg) prm.SetNodeDialTimeout(connTimeout) prmTree.SetNodeDialTimeout(connTimeout) - streamTimeout := fetchStreamTimeout(cfg) + streamTimeout := fetchStreamTimeout(a.cfg) prm.SetNodeStreamTimeout(streamTimeout) prmTree.SetNodeStreamTimeout(streamTimeout) - healthCheckTimeout := fetchHealthCheckTimeout(cfg) + healthCheckTimeout := fetchHealthCheckTimeout(a.cfg) prm.SetHealthcheckTimeout(healthCheckTimeout) prmTree.SetHealthcheckTimeout(healthCheckTimeout) - rebalanceInterval := fetchRebalanceInterval(cfg) + rebalanceInterval := fetchRebalanceInterval(a.cfg) prm.SetClientRebalanceInterval(rebalanceInterval) prmTree.SetClientRebalanceInterval(rebalanceInterval) - errorThreshold := fetchErrorThreshold(cfg) + errorThreshold := fetchErrorThreshold(a.cfg) prm.SetErrorThreshold(errorThreshold) - prm.SetGracefulCloseOnSwitchTimeout(fetchSetGracefulCloseOnSwitchTimeout(cfg)) + prm.SetGracefulCloseOnSwitchTimeout(fetchSetGracefulCloseOnSwitchTimeout(a.cfg)) - prm.SetLogger(logger) - prmTree.SetLogger(logger) + prm.SetLogger(a.log) + prmTree.SetLogger(a.log) - prmTree.SetMaxRequestAttempts(cfg.GetInt(cfgTreePoolMaxAttempts)) + prmTree.SetMaxRequestAttempts(a.cfg.GetInt(cfgTreePoolMaxAttempts)) interceptors := []grpc.DialOption{ grpc.WithUnaryInterceptor(grpctracing.NewUnaryClientInteceptor()), grpc.WithStreamInterceptor(grpctracing.NewStreamClientInterceptor()), - grpc.WithContextDialer(dialSource.GrpcContextDialer()), + grpc.WithContextDialer(a.settings.dialerSource.GrpcContextDialer()), } prm.SetGRPCDialOptions(interceptors...) prmTree.SetGRPCDialOptions(interceptors...) p, err := pool.NewPool(prm) if err != nil { - logger.Fatal(logs.FailedToCreateConnectionPool, zap.Error(err)) + a.log.Fatal(logs.FailedToCreateConnectionPool, zap.Error(err)) } if err = p.Dial(ctx); err != nil { - logger.Fatal(logs.FailedToDialConnectionPool, zap.Error(err)) + a.log.Fatal(logs.FailedToDialConnectionPool, zap.Error(err)) + } + + if a.cfg.GetBool(cfgTreePoolNetmapSupport) { + prmTree.SetNetMapInfoSource(frostfs.NewSource(frostfs.NewFrostFS(p, key), a.cache)) } treePool, err := treepool.NewPool(prmTree) if err != nil { - logger.Fatal(logs.FailedToCreateTreePool, zap.Error(err)) + a.log.Fatal(logs.FailedToCreateTreePool, zap.Error(err)) } if err = treePool.Dial(ctx); err != nil { - logger.Fatal(logs.FailedToDialTreePool, zap.Error(err)) + a.log.Fatal(logs.FailedToDialTreePool, zap.Error(err)) } - return p, treePool, key + a.treePool = treePool + a.pool = p + a.key = key } func remove(list []string, element string) []string { @@ -1091,7 +1096,9 @@ func getCacheOptions(v *viper.Viper, l *zap.Logger) *layer.CachesConfig { cacheCfg.AccessControl.Lifetime = fetchCacheLifetime(v, l, cfgAccessControlCacheLifetime, cacheCfg.AccessControl.Lifetime) cacheCfg.AccessControl.Size = fetchCacheSize(v, l, cfgAccessControlCacheSize, cacheCfg.AccessControl.Size) - cacheCfg.NetworkInfo.Lifetime = fetchCacheLifetime(v, l, cfgNetworkInfoCacheLifetime, cacheCfg.NetworkInfo.Lifetime) + cacheCfg.Network.Lifetime = fetchCacheLifetime(v, l, cfgNetworkCacheLifetime, cacheCfg.Network.Lifetime) + + cacheCfg.CIDCache = v.GetBool(cfgTreePoolNetmapSupport) return cacheCfg } diff --git a/cmd/s3-gw/app_settings.go b/cmd/s3-gw/app_settings.go index 8376966..87e715c 100644 --- a/cmd/s3-gw/app_settings.go +++ b/cmd/s3-gw/app_settings.go @@ -147,7 +147,7 @@ const ( // Settings. cfgMorphPolicyCacheSize = "cache.morph_policy.size" cfgFrostfsIDCacheLifetime = "cache.frostfsid.lifetime" cfgFrostfsIDCacheSize = "cache.frostfsid.size" - cfgNetworkInfoCacheLifetime = "cache.network_info.lifetime" + cfgNetworkCacheLifetime = "cache.network_info.lifetime" cfgAccessBoxCacheRemovingCheckInterval = "cache.accessbox.removing_check_interval" @@ -268,8 +268,9 @@ const ( // Settings. cfgSoftMemoryLimit = "runtime.soft_memory_limit" // Enable return MD5 checksum in ETag. - cfgMD5Enabled = "features.md5.enabled" - cfgPolicyDenyByDefault = "features.policy.deny_by_default" + cfgMD5Enabled = "features.md5.enabled" + cfgPolicyDenyByDefault = "features.policy.deny_by_default" + cfgTreePoolNetmapSupport = "features.tree_pool_netmap_support" // FrostfsID. cfgFrostfsIDContract = "frostfsid.contract" diff --git a/config/config.env b/config/config.env index 9ffe8c7..1e3268e 100644 --- a/config/config.env +++ b/config/config.env @@ -126,7 +126,7 @@ S3_GW_CACHE_MORPH_POLICY_SIZE=10000 # Cache which stores frostfsid subject info S3_GW_CACHE_FROSTFSID_LIFETIME=1m S3_GW_CACHE_FROSTFSID_SIZE=10000 -# Cache which stores network info +# Cache which stores network-related values S3_GW_CACHE_NETWORK_INFO_LIFETIME=1m # Default policy of placing containers in FrostFS @@ -205,6 +205,8 @@ S3_GW_RUNTIME_SOFT_MEMORY_LIMIT=1073741824 S3_GW_FEATURES_MD5_ENABLED=false # Enable denying access for request that doesn't match any policy chain rules. S3_GW_FEATURES_POLICY_DENY_BY_DEFAULT=false +# Enable using new version of tree pool, which uses netmap to select nodes, for requests to tree service +S3_GW_FEATURES_TREE_POOL_NETMAP_SUPPORT=true # ReadTimeout is the maximum duration for reading the entire # request, including the body. A zero or negative value means diff --git a/config/config.yaml b/config/config.yaml index 834aa1b..b3f86f8 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -158,7 +158,7 @@ cache: frostfsid: lifetime: 1m size: 10000 - # Cache which stores network info + # Cache which stores network-related values network_info: lifetime: 1m @@ -243,6 +243,8 @@ features: deny_by_default: false md5: enabled: false + # Enable using new version of tree pool, which uses netmap to select nodes, for requests to tree service + tree_pool_netmap_support: true web: # ReadTimeout is the maximum duration for reading the entire diff --git a/docs/configuration.md b/docs/configuration.md index e100855..d4417b2 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -465,7 +465,7 @@ cache: | `accesscontrol` | [Cache config](#cache-subsection) | `lifetime: 1m`
`size: 100000` | Cache which stores owner to cache operation mapping. | | `morph_policy` | [Cache config](#cache-subsection) | `lifetime: 1m`
`size: 10000` | Cache which stores list of policy chains. | | `frostfsid` | [Cache config](#cache-subsection) | `lifetime: 1m`
`size: 10000` | Cache which stores FrostfsID subject info. | -| `network_info` | [Cache config](#cache-subsection) | `lifetime: 1m` | Cache which stores network info. | +| `network_info` | [Cache config](#cache-subsection) | `lifetime: 1m` | Cache which stores network-related values. | #### `cache` subsection @@ -688,12 +688,14 @@ features: deny_by_default: false md5: enabled: false + tree_pool_netmap_support: true ``` -| Parameter | Type | SIGHUP reload | Default value | Description | -|--------------------------|--------|---------------|---------------|------------------------------------------------------------------------------| -| `md5.enabled` | `bool` | yes | false | Flag to enable return MD5 checksum in ETag headers and fields. | -| `policy.deny_by_default` | `bool` | yes | false | Enable denying access for request that doesn't match any policy chain rules. | +| Parameter | Type | SIGHUP reload | Default value | Description | +|----------------------------|--------|---------------|---------------|---------------------------------------------------------------------------------------------------------| +| `md5.enabled` | `bool` | yes | false | Flag to enable return MD5 checksum in ETag headers and fields. | +| `policy.deny_by_default` | `bool` | yes | false | Enable denying access for request that doesn't match any policy chain rules. | +| `tree_pool_netmap_support` | `bool` | no | false | Enable using new version of tree pool, which uses netmap to select nodes, for requests to tree service. | # `web` section Contains web server configuration parameters. diff --git a/go.mod b/go.mod index 2d748d8..e44194f 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.22 require ( git.frostfs.info/TrueCloudLab/frostfs-contract v0.20.1-0.20241022094040-5f956751d48b git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241212101224-902f32eeabcf + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241218062344-42a0fc8c13ae git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240822104152-a3bc3099bd5b git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 @@ -74,10 +74,19 @@ require ( github.com/hashicorp/hcl v1.0.0 // indirect github.com/holiman/uint256 v1.2.4 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/ipfs/go-cid v0.0.7 // indirect github.com/josharian/intern v1.0.0 // indirect + github.com/klauspost/cpuid/v2 v2.2.6 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect + github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/multiformats/go-base32 v0.1.0 // indirect + github.com/multiformats/go-base36 v0.2.0 // indirect + github.com/multiformats/go-multiaddr v0.14.0 // indirect + github.com/multiformats/go-multibase v0.2.0 // indirect + github.com/multiformats/go-multihash v0.2.3 // indirect + github.com/multiformats/go-varint v0.0.7 // indirect github.com/nspcc-dev/go-ordered-json v0.0.0-20240301084351-0246b013f8b2 // indirect github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20240727093519-1a48f1ce43ec // indirect github.com/nspcc-dev/rfc6979 v0.2.1 // indirect @@ -87,6 +96,7 @@ require ( github.com/prometheus/common v0.48.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/afero v1.9.3 // indirect github.com/spf13/cast v1.5.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect @@ -108,4 +118,5 @@ require ( google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + lukechampine.com/blake3 v1.2.1 // indirect ) diff --git a/go.sum b/go.sum index 370e793..7fb8068 100644 --- a/go.sum +++ b/go.sum @@ -42,8 +42,8 @@ git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSV git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88 h1:9bvBDLApbbO5sXBKdODpE9tzy3HV99nXxkDWNn22rdI= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241212101224-902f32eeabcf h1:PqRKQX+Xlqq4qsAlr7Jcx2kapLdPGZZQ09MEW+ui2c4= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241212101224-902f32eeabcf/go.mod h1:eoK7+KZQ9GJxbzIs6vTnoUJqFDppavInLRHaN4MYgZg= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241218062344-42a0fc8c13ae h1:7gvuOTmS3oaOM79JkHWWlsvGqIRqsum5KnOI1TYqfn0= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241218062344-42a0fc8c13ae/go.mod h1:dbWUc5jOBTXVvssCLCYxkkSTL9jgLr1KruGP2FMAfiM= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 h1:/960fWeyn2AFHwQUwDsWB3sbP6lTEnFnMzLMM6tx6N8= @@ -216,11 +216,15 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/ipfs/go-cid v0.0.7 h1:ysQJVJA3fNDF1qigJbsSQOdjhVLsOEoPdh0+R97k3jY= +github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= +github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -233,14 +237,37 @@ github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0V github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= +github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= +github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/minio/sio v0.3.0 h1:syEFBewzOMOYVzSTFpp1MqpSZk8rUNbz8VIIc+PNzus= github.com/minio/sio v0.3.0/go.mod h1:8b0yPp2avGThviy/+OCJBI6OMpvxoUuiLvE6F1lebhw= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mmcloughlin/addchain v0.4.0 h1:SobOdjm2xLj1KkXN5/n0xTIWyZA2+s99UCY1iPfkHRY= github.com/mmcloughlin/addchain v0.4.0/go.mod h1:A86O+tHqZLMNO4w6ZZ4FlVQEadcoqkyU72HC5wJ4RlU= +github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= +github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= +github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA= +github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aGkbLYxPE= +github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYgtWibDcT0rExnbI= +github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM= +github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0= +github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4= +github.com/multiformats/go-multiaddr v0.14.0 h1:bfrHrJhrRuh/NXH5mCnemjpbGjzRw/b+tJFOD41g2tU= +github.com/multiformats/go-multiaddr v0.14.0/go.mod h1:6EkVAxtznq2yC3QT5CM1UTAwG0GTP3EWAIcjHuzQ+r4= +github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= +github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g= +github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6oPyjtnzK/XTFDPkNuk= +github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= +github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U= +github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM= +github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= +github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8= +github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= github.com/nspcc-dev/dbft v0.2.0 h1:sDwsQES600OSIMncV176t2SX5OvB14lzeOAyKFOkbMI= github.com/nspcc-dev/dbft v0.2.0/go.mod h1:oFE6paSC/yfFh9mcNU6MheMGOYXK9+sPiRk3YMoz49o= github.com/nspcc-dev/go-ordered-json v0.0.0-20240301084351-0246b013f8b2 h1:mD9hU3v+zJcnHAVmHnZKt3I++tvn30gBj2rP2PocZMk= @@ -284,6 +311,8 @@ github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.9.3 h1:41FoI0fD7OR7mGcKE/aOiLkGreyf8ifIOQmJANWogMk= github.com/spf13/afero v1.9.3/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= @@ -362,6 +391,7 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= @@ -504,6 +534,7 @@ golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -699,6 +730,8 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +lukechampine.com/blake3 v1.2.1 h1:YuqqRuaqsGV71BV/nm9xlI0MKUv4QC54jQnBChWbGnI= +lukechampine.com/blake3 v1.2.1/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/internal/frostfs/frostfs.go b/internal/frostfs/frostfs.go index 7b4fba9..186d206 100644 --- a/internal/frostfs/frostfs.go +++ b/internal/frostfs/frostfs.go @@ -409,6 +409,15 @@ func (x *FrostFS) NetworkInfo(ctx context.Context) (netmap.NetworkInfo, error) { return ni, nil } +func (x *FrostFS) NetmapSnapshot(ctx context.Context) (netmap.NetMap, error) { + netmapSnapshot, err := x.pool.NetMapSnapshot(ctx) + if err != nil { + return netmapSnapshot, handleObjectError("get netmap via connection pool", err) + } + + return netmapSnapshot, nil +} + func (x *FrostFS) PatchObject(ctx context.Context, prm frostfs.PrmObjectPatch) (oid.ID, error) { var addr oid.Address addr.SetContainer(prm.Container) diff --git a/internal/frostfs/source.go b/internal/frostfs/source.go new file mode 100644 index 0000000..bf1d0af --- /dev/null +++ b/internal/frostfs/source.go @@ -0,0 +1,66 @@ +package frostfs + +import ( + "context" + "fmt" + + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/frostfs" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" +) + +type Source struct { + frostFS *FrostFS + cache *layer.Cache +} + +func NewSource(frostFS *FrostFS, cache *layer.Cache) *Source { + return &Source{ + frostFS: frostFS, + cache: cache, + } +} + +func (s *Source) NetMapSnapshot(ctx context.Context) (netmap.NetMap, error) { + cachedNetmap := s.cache.GetNetmap() + if cachedNetmap != nil { + return *cachedNetmap, nil + } + + netmapSnapshot, err := s.frostFS.NetmapSnapshot(ctx) + if err != nil { + return netmap.NetMap{}, fmt.Errorf("get netmap: %w", err) + } + + s.cache.PutNetmap(netmapSnapshot) + + return netmapSnapshot, nil +} + +func (s *Source) PlacementPolicy(ctx context.Context, cnrID cid.ID) (netmap.PlacementPolicy, error) { + cachedPolicy := s.cache.GetPlacementPolicy(cnrID) + if cachedPolicy != nil { + return *cachedPolicy, nil + } + + prm := frostfs.PrmContainer{ + ContainerID: cnrID, + } + if bd, err := middleware.GetBoxData(ctx); err == nil && bd.Gate != nil { + prm.SessionToken = bd.Gate.SessionToken() + } + + res, err := s.frostFS.Container(ctx, prm) + if err != nil { + return netmap.PlacementPolicy{}, fmt.Errorf("get container: %w", err) + } + + // We don't put container back to the cache to keep cache + // coherent to the requests made by S3 users. FrostFS Source + // is being used by SDK Tree Pool and it should not fill cache + // with possibly irrelevant container values. + + return res.PlacementPolicy(), nil +} diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 49752b1..00c0d73 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -184,4 +184,5 @@ const ( WarnInvalidTypeTLSTerminationHeader = "invalid type of value of tls termination header" FailedToPutTombstones = "failed to put tombstones" WarnDomainContainsPort = "the domain contains a port, domain skipped" + CouldntCacheNetmap = "couldn't cache netmap" )