diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index 23a752a..3386536 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -65,6 +65,7 @@ type ( services []*metrics.Service settings *appSettings loggerSettings *loggerSettings + bucketCache *cache.BucketCache servers []Server unbindServers []ServerInfo @@ -134,6 +135,7 @@ func newApp(ctx context.Context, v *viper.Viper) App { loggerSettings: logSettings, webServer: new(fasthttp.Server), webDone: make(chan struct{}), + bucketCache: cache.NewBucketCache(getBucketCacheOptions(v, log.logger), v.GetBool(cfgFeaturesTreePoolNetmapSupport)), } a.initAppSettings() @@ -151,7 +153,7 @@ func newApp(ctx context.Context, v *viper.Viper) App { a.webServer.DisablePreParseMultipartForm = true a.webServer.StreamRequestBody = a.cfg.GetBool(cfgWebStreamRequestBody) // -- -- -- -- -- -- -- -- -- -- -- -- -- -- - a.pool, a.treePool, a.key = getPools(ctx, a.log, a.cfg, a.settings.dialerSource) + a.initPools(ctx) var owner user.ID user.IDFromKey(&owner, a.key.PrivateKey.PublicKey) @@ -839,7 +841,7 @@ func (a *app) AppParams() *handler.AppParams { FrostFS: frostfs.NewFrostFS(a.pool), Owner: a.owner, Resolver: a.resolver, - Cache: cache.NewBucketCache(getCacheOptions(a.cfg, a.log)), + Cache: a.bucketCache, } } diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index 62ef83e..691e9ba 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -17,12 +17,12 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" internalnet "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/net" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/service/frostfs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" grpctracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" treepool "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree" "git.frostfs.info/TrueCloudLab/zapjournald" - "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/spf13/pflag" "github.com/spf13/viper" "github.com/ssgreg/journald" @@ -144,6 +144,7 @@ const ( // Caching. cfgBucketsCacheLifetime = "cache.buckets.lifetime" cfgBucketsCacheSize = "cache.buckets.size" + cfgNetmapCacheLifetime = "cache.netmap.lifetime" // Bucket resolving options. cfgResolveNamespaceHeader = "resolve_bucket.namespace_header" @@ -166,6 +167,7 @@ const ( // Feature. cfgFeaturesEnableFilepathFallback = "features.enable_filepath_fallback" + cfgFeaturesTreePoolNetmapSupport = "features.tree_pool_netmap_support" // Command line args. cmdHelp = "help" @@ -611,10 +613,10 @@ func fetchServers(v *viper.Viper, log *zap.Logger) []ServerInfo { return servers } -func getPools(ctx context.Context, logger *zap.Logger, cfg *viper.Viper, dialSource *internalnet.DialerSource) (*pool.Pool, *treepool.Pool, *keys.PrivateKey) { - key, err := getFrostFSKey(cfg, logger) +func (a *app) initPools(ctx context.Context) { + key, err := getFrostFSKey(a.cfg, a.log) if err != nil { - logger.Fatal(logs.CouldNotLoadFrostFSPrivateKey, zap.Error(err)) + a.log.Fatal(logs.CouldNotLoadFrostFSPrivateKey, zap.Error(err)) } var prm pool.InitParameters @@ -622,77 +624,83 @@ func getPools(ctx context.Context, logger *zap.Logger, cfg *viper.Viper, dialSou prm.SetKey(&key.PrivateKey) prmTree.SetKey(key) - logger.Info(logs.UsingCredentials, zap.String("FrostFS", hex.EncodeToString(key.PublicKey().Bytes()))) + a.log.Info(logs.UsingCredentials, zap.String("FrostFS", hex.EncodeToString(key.PublicKey().Bytes()))) - for _, peer := range fetchPeers(logger, cfg) { + for _, peer := range fetchPeers(a.log, a.cfg) { prm.AddNode(peer) prmTree.AddNode(peer) } - connTimeout := cfg.GetDuration(cfgConTimeout) + connTimeout := a.cfg.GetDuration(cfgConTimeout) if connTimeout <= 0 { connTimeout = defaultConnectTimeout } prm.SetNodeDialTimeout(connTimeout) prmTree.SetNodeDialTimeout(connTimeout) - streamTimeout := cfg.GetDuration(cfgStreamTimeout) + streamTimeout := a.cfg.GetDuration(cfgStreamTimeout) if streamTimeout <= 0 { streamTimeout = defaultStreamTimeout } prm.SetNodeStreamTimeout(streamTimeout) prmTree.SetNodeStreamTimeout(streamTimeout) - healthCheckTimeout := cfg.GetDuration(cfgReqTimeout) + healthCheckTimeout := a.cfg.GetDuration(cfgReqTimeout) if healthCheckTimeout <= 0 { healthCheckTimeout = defaultRequestTimeout } prm.SetHealthcheckTimeout(healthCheckTimeout) prmTree.SetHealthcheckTimeout(healthCheckTimeout) - rebalanceInterval := cfg.GetDuration(cfgRebalance) + rebalanceInterval := a.cfg.GetDuration(cfgRebalance) if rebalanceInterval <= 0 { rebalanceInterval = defaultRebalanceTimer } prm.SetClientRebalanceInterval(rebalanceInterval) prmTree.SetClientRebalanceInterval(rebalanceInterval) - errorThreshold := cfg.GetUint32(cfgPoolErrorThreshold) + errorThreshold := a.cfg.GetUint32(cfgPoolErrorThreshold) if errorThreshold <= 0 { errorThreshold = defaultPoolErrorThreshold } prm.SetErrorThreshold(errorThreshold) - prm.SetLogger(logger) - prmTree.SetLogger(logger) + prm.SetLogger(a.log) + prmTree.SetLogger(a.log) - prmTree.SetMaxRequestAttempts(cfg.GetInt(cfgTreePoolMaxAttempts)) + prmTree.SetMaxRequestAttempts(a.cfg.GetInt(cfgTreePoolMaxAttempts)) interceptors := []grpc.DialOption{ grpc.WithUnaryInterceptor(grpctracing.NewUnaryClientInteceptor()), grpc.WithStreamInterceptor(grpctracing.NewStreamClientInterceptor()), - grpc.WithContextDialer(dialSource.GrpcContextDialer()), + grpc.WithContextDialer(a.settings.dialerSource.GrpcContextDialer()), } prm.SetGRPCDialOptions(interceptors...) prmTree.SetGRPCDialOptions(interceptors...) p, err := pool.NewPool(prm) if err != nil { - logger.Fatal(logs.FailedToCreateConnectionPool, zap.Error(err)) + a.log.Fatal(logs.FailedToCreateConnectionPool, zap.Error(err)) } if err = p.Dial(ctx); err != nil { - logger.Fatal(logs.FailedToDialConnectionPool, zap.Error(err)) + a.log.Fatal(logs.FailedToDialConnectionPool, zap.Error(err)) + } + + if a.cfg.GetBool(cfgFeaturesTreePoolNetmapSupport) { + prmTree.SetNetMapInfoSource(frostfs.NewSource(frostfs.NewFrostFS(p), cache.NewNetmapCache(getNetmapCacheOptions(a.cfg, a.log)), a.bucketCache, a.log)) } treePool, err := treepool.NewPool(prmTree) if err != nil { - logger.Fatal(logs.FailedToCreateTreePool, zap.Error(err)) + a.log.Fatal(logs.FailedToCreateTreePool, zap.Error(err)) } if err = treePool.Dial(ctx); err != nil { - logger.Fatal(logs.FailedToDialTreePool, zap.Error(err)) + a.log.Fatal(logs.FailedToDialTreePool, zap.Error(err)) } - return p, treePool, key + a.pool = p + a.treePool = treePool + a.key = key } func fetchPeers(l *zap.Logger, v *viper.Viper) []pool.NodeParam { @@ -733,7 +741,7 @@ func fetchSoftMemoryLimit(cfg *viper.Viper) int64 { return int64(softMemoryLimit) } -func getCacheOptions(v *viper.Viper, l *zap.Logger) *cache.Config { +func getBucketCacheOptions(v *viper.Viper, l *zap.Logger) *cache.Config { cacheCfg := cache.DefaultBucketConfig(l) cacheCfg.Lifetime = fetchCacheLifetime(v, l, cfgBucketsCacheLifetime, cacheCfg.Lifetime) @@ -742,6 +750,14 @@ func getCacheOptions(v *viper.Viper, l *zap.Logger) *cache.Config { return cacheCfg } +func getNetmapCacheOptions(v *viper.Viper, l *zap.Logger) *cache.NetmapCacheConfig { + cacheCfg := cache.DefaultNetmapConfig(l) + + cacheCfg.Lifetime = fetchCacheLifetime(v, l, cfgNetmapCacheLifetime, cacheCfg.Lifetime) + + return cacheCfg +} + func fetchCacheLifetime(v *viper.Viper, l *zap.Logger, cfgEntry string, defaultValue time.Duration) time.Duration { if v.IsSet(cfgEntry) { lifetime := v.GetDuration(cfgEntry) diff --git a/config/config.env b/config/config.env index 2822357..171889f 100644 --- a/config/config.env +++ b/config/config.env @@ -121,6 +121,8 @@ HTTP_GW_FROSTFS_BUFFER_MAX_SIZE_FOR_PUT=1048576 # Cache which contains mapping of bucket name to bucket info HTTP_GW_CACHE_BUCKETS_LIFETIME=1m HTTP_GW_CACHE_BUCKETS_SIZE=1000 +# Cache which stores netmap +HTTP_GW_CACHE_NETMAP_LIFETIME=1m # Header to determine zone to resolve bucket name HTTP_GW_RESOLVE_BUCKET_NAMESPACE_HEADER=X-Frostfs-Namespace @@ -162,3 +164,5 @@ HTTP_GW_INDEX_PAGE_TEMPLATE_PATH=internal/handler/templates/index.gotmpl # Enable using fallback path to search for a object by attribute HTTP_GW_FEATURES_ENABLE_FILEPATH_FALLBACK=false +# Enable using new version of tree pool, which uses netmap to select nodes, for requests to tree service +HTTP_GW_FEATURES_TREE_POOL_NETMAP_SUPPORT=true diff --git a/config/config.yaml b/config/config.yaml index 6296bd9..eee84e5 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -143,6 +143,9 @@ cache: buckets: lifetime: 1m size: 1000 + # Cache which stores netmap + netmap: + lifetime: 1m resolve_bucket: namespace_header: X-Frostfs-Namespace @@ -176,3 +179,5 @@ multinet: features: # Enable using fallback path to search for a object by attribute enable_filepath_fallback: false + # Enable using new version of tree pool, which uses netmap to select nodes, for requests to tree service + tree_pool_netmap_support: true diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index 7476f5d..c1772f4 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -339,12 +339,14 @@ cache: buckets: lifetime: 1m size: 1000 - + netmap: + lifetime: 1m ``` -| Parameter | Type | Default value | Description | -|-----------------|-----------------------------------|-----------------------------------|----------------------------------------------------------------------------------------| -| `buckets` | [Cache config](#cache-subsection) | `lifetime: 60s`
`size: 1000` | Cache which contains mapping of bucket name to bucket info. | +| Parameter | Type | Default value | Description | +|-----------|-----------------------------------|---------------------------------|-------------------------------------------------------------| +| `buckets` | [Cache config](#cache-subsection) | `lifetime: 60s`
`size: 1000` | Cache which contains mapping of bucket name to bucket info. | +| `netmap` | [Cache config](#cache-subsection) | `lifetime: 1m` | Cache which stores netmap. | #### `cache` subsection @@ -465,8 +467,10 @@ Contains parameters for enabling features. ```yaml features: enable_filepath_fallback: true + tree_pool_netmap_support: true ``` | Parameter | Type | SIGHUP reload | Default value | Description | -| ----------------------------------- | ------ | ------------- | ------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +|-------------------------------------|--------|---------------|---------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `features.enable_filepath_fallback` | `bool` | yes | `false` | Enable using fallback path to search for a object by attribute. If the value of the `FilePath` attribute in the request contains no `/` symbols or single leading `/` symbol and the object was not found, then an attempt is made to search for the object by the attribute `FileName`. | +| `features.tree_pool_netmap_support` | `bool` | no | `false` | Enable using new version of tree pool, which uses netmap to select nodes, for requests to tree service. | diff --git a/go.mod b/go.mod index 3dd27b8..0b74841 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.22 require ( git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241206094944-81c423e7094d + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241218062344-42a0fc8c13ae git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 github.com/bluele/gcache v0.0.2 @@ -65,16 +65,25 @@ require ( github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/hashicorp/hcl v1.0.0 // indirect + github.com/ipfs/go-cid v0.0.7 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/klauspost/compress v1.16.4 // indirect + github.com/klauspost/cpuid/v2 v2.2.6 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect + github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/moby/sys/mount v0.3.2 // indirect github.com/moby/sys/mountinfo v0.6.1 // indirect github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect github.com/morikuni/aec v1.0.0 // indirect github.com/mr-tron/base58 v1.2.0 // indirect + github.com/multiformats/go-base32 v0.1.0 // indirect + github.com/multiformats/go-base36 v0.2.0 // indirect + github.com/multiformats/go-multiaddr v0.14.0 // indirect + github.com/multiformats/go-multibase v0.2.0 // indirect + github.com/multiformats/go-multihash v0.2.3 // indirect + github.com/multiformats/go-varint v0.0.7 // indirect github.com/nspcc-dev/go-ordered-json v0.0.0-20240301084351-0246b013f8b2 // indirect github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20240521091047-78685785716d // indirect github.com/nspcc-dev/rfc6979 v0.2.1 // indirect @@ -89,6 +98,7 @@ require ( github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/savsgio/gotils v0.0.0-20210617111740-97865ed5a873 // indirect github.com/sirupsen/logrus v1.8.1 // indirect + github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/afero v1.9.3 // indirect github.com/spf13/cast v1.5.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect @@ -116,4 +126,5 @@ require ( google.golang.org/protobuf v1.34.2 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + lukechampine.com/blake3 v1.2.1 // indirect ) diff --git a/go.sum b/go.sum index 7f43c86..06bdd7d 100644 --- a/go.sum +++ b/go.sum @@ -43,8 +43,8 @@ git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSV git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88 h1:9bvBDLApbbO5sXBKdODpE9tzy3HV99nXxkDWNn22rdI= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241206094944-81c423e7094d h1:FpXI+mOrmJk3t2MKQFZuhLjCHDyDeo5rtP1WXl7gUWc= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241206094944-81c423e7094d/go.mod h1:eoK7+KZQ9GJxbzIs6vTnoUJqFDppavInLRHaN4MYgZg= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241218062344-42a0fc8c13ae h1:7gvuOTmS3oaOM79JkHWWlsvGqIRqsum5KnOI1TYqfn0= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241218062344-42a0fc8c13ae/go.mod h1:dbWUc5jOBTXVvssCLCYxkkSTL9jgLr1KruGP2FMAfiM= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 h1:/960fWeyn2AFHwQUwDsWB3sbP6lTEnFnMzLMM6tx6N8= @@ -519,6 +519,8 @@ github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/ipfs/go-cid v0.0.7 h1:ysQJVJA3fNDF1qigJbsSQOdjhVLsOEoPdh0+R97k3jY= +github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= @@ -544,6 +546,8 @@ github.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8 github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.16.4 h1:91KN02FnsOYhuunwU4ssRe8lc2JosWmizWa91B5v1PU= github.com/klauspost/compress v1.16.4/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= +github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -582,6 +586,10 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY= github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= +github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= +github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= +github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= @@ -610,9 +618,28 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= +github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= +github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA= +github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aGkbLYxPE= +github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYgtWibDcT0rExnbI= +github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM= +github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0= +github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4= +github.com/multiformats/go-multiaddr v0.14.0 h1:bfrHrJhrRuh/NXH5mCnemjpbGjzRw/b+tJFOD41g2tU= +github.com/multiformats/go-multiaddr v0.14.0/go.mod h1:6EkVAxtznq2yC3QT5CM1UTAwG0GTP3EWAIcjHuzQ+r4= +github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= +github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g= +github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6oPyjtnzK/XTFDPkNuk= +github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= +github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U= +github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM= +github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= +github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8= +github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -762,6 +789,8 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1 github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/afero v1.9.3 h1:41FoI0fD7OR7mGcKE/aOiLkGreyf8ifIOQmJANWogMk= @@ -1121,6 +1150,7 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -1407,6 +1437,8 @@ k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +lukechampine.com/blake3 v1.2.1 h1:YuqqRuaqsGV71BV/nm9xlI0MKUv4QC54jQnBChWbGnI= +lukechampine.com/blake3 v1.2.1/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/internal/cache/buckets.go b/internal/cache/buckets.go index f8e6d88..2fa8f25 100644 --- a/internal/cache/buckets.go +++ b/internal/cache/buckets.go @@ -6,14 +6,16 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "github.com/bluele/gcache" "go.uber.org/zap" ) // BucketCache contains cache with objects and the lifetime of cache entries. type BucketCache struct { - cache gcache.Cache - logger *zap.Logger + cache gcache.Cache + cidCache gcache.Cache + logger *zap.Logger } // Config stores expiration params for cache. @@ -40,14 +42,45 @@ func DefaultBucketConfig(logger *zap.Logger) *Config { } // NewBucketCache creates an object of BucketCache. -func NewBucketCache(config *Config) *BucketCache { - gc := gcache.New(config.Size).LRU().Expiration(config.Lifetime).Build() - return &BucketCache{cache: gc, logger: config.Logger} +func NewBucketCache(config *Config, cidCache bool) *BucketCache { + cache := &BucketCache{ + cache: gcache.New(config.Size).LRU().Expiration(config.Lifetime).Build(), + logger: config.Logger, + } + + if cidCache { + cache.cidCache = gcache.New(config.Size).LRU().Expiration(config.Lifetime).Build() + } + return cache } // Get returns a cached object. func (o *BucketCache) Get(ns, bktName string) *data.BucketInfo { - entry, err := o.cache.Get(formKey(ns, bktName)) + return o.get(formKey(ns, bktName)) +} + +func (o *BucketCache) GetByCID(cnrID cid.ID) *data.BucketInfo { + if o.cidCache == nil { + return nil + } + + entry, err := o.cidCache.Get(cnrID) + if err != nil { + return nil + } + + key, ok := entry.(string) + if !ok { + o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)), + zap.String("expected", fmt.Sprintf("%T", key))) + return nil + } + + return o.get(key) +} + +func (o *BucketCache) get(key string) *data.BucketInfo { + entry, err := o.cache.Get(key) if err != nil { return nil } @@ -64,6 +97,12 @@ func (o *BucketCache) Get(ns, bktName string) *data.BucketInfo { // Put puts an object to cache. func (o *BucketCache) Put(bkt *data.BucketInfo) error { + if o.cidCache != nil { + if err := o.cidCache.Set(bkt.CID, formKey(bkt.Zone, bkt.Name)); err != nil { + return err + } + } + return o.cache.Set(formKey(bkt.Zone, bkt.Name), bkt) } diff --git a/internal/cache/netmap.go b/internal/cache/netmap.go new file mode 100644 index 0000000..e092060 --- /dev/null +++ b/internal/cache/netmap.go @@ -0,0 +1,65 @@ +package cache + +import ( + "fmt" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" + "github.com/bluele/gcache" + "go.uber.org/zap" +) + +type ( + // NetmapCache provides cache for netmap. + NetmapCache struct { + cache gcache.Cache + logger *zap.Logger + } + + // NetmapCacheConfig stores expiration params for cache. + NetmapCacheConfig struct { + Lifetime time.Duration + Logger *zap.Logger + } +) + +const ( + DefaultNetmapCacheLifetime = 1 * time.Minute + netmapCacheSize = 1 + netmapKey = "netmap" +) + +// DefaultNetmapConfig returns new default cache expiration values. +func DefaultNetmapConfig(logger *zap.Logger) *NetmapCacheConfig { + return &NetmapCacheConfig{ + Lifetime: DefaultNetmapCacheLifetime, + Logger: logger, + } +} + +// NewNetmapCache creates an object of NetmapCache. +func NewNetmapCache(config *NetmapCacheConfig) *NetmapCache { + gc := gcache.New(netmapCacheSize).LRU().Expiration(config.Lifetime).Build() + return &NetmapCache{cache: gc, logger: config.Logger} +} + +func (c *NetmapCache) Get() *netmap.NetMap { + entry, err := c.cache.Get(netmapKey) + if err != nil { + return nil + } + + result, ok := entry.(netmap.NetMap) + if !ok { + c.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)), + zap.String("expected", fmt.Sprintf("%T", result))) + return nil + } + + return &result +} + +func (c *NetmapCache) Put(nm netmap.NetMap) error { + return c.cache.Set(netmapKey, nm) +} diff --git a/internal/data/info.go b/internal/data/info.go index d99ca49..f5c80d6 100644 --- a/internal/data/info.go +++ b/internal/data/info.go @@ -2,6 +2,7 @@ package data import ( cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" ) type BucketInfo struct { @@ -9,4 +10,5 @@ type BucketInfo struct { Zone string // container zone from system attribute CID cid.ID HomomorphicHashDisabled bool + PlacementPolicy netmap.PlacementPolicy } diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 3805c2d..1150f45 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -365,6 +365,7 @@ func (h *Handler) readContainer(ctx context.Context, cnrID cid.ID) (*data.Bucket } bktInfo.HomomorphicHashDisabled = container.IsHomomorphicHashingDisabled(*res) + bktInfo.PlacementPolicy = res.PlacementPolicy() return bktInfo, err } diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index 14f9c98..e1bc010 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -142,7 +142,7 @@ func prepareHandlerContext() (*handlerContext, error) { Size: 1, Lifetime: 1, Logger: logger, - }), + }, false), } treeMock := newTreeService() diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 7a04064..f9b13b1 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -93,4 +93,5 @@ const ( FailedToLoadMultinetConfig = "failed to load multinet config" MultinetConfigWontBeUpdated = "multinet config won't be updated" ObjectNotFoundByFilePathTrySearchByFileName = "object not found by filePath attribute, try search by fileName" + CouldntCacheNetmap = "couldn't cache netmap" ) diff --git a/internal/service/frostfs/frostfs.go b/internal/service/frostfs/frostfs.go index c7e56a4..b218976 100644 --- a/internal/service/frostfs/frostfs.go +++ b/internal/service/frostfs/frostfs.go @@ -11,6 +11,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" @@ -173,6 +174,15 @@ func (x *FrostFS) GetEpochDurations(ctx context.Context) (*utils.EpochDurations, return res, nil } +func (x *FrostFS) NetmapSnapshot(ctx context.Context) (netmap.NetMap, error) { + netmapSnapshot, err := x.pool.NetMapSnapshot(ctx) + if err != nil { + return netmapSnapshot, handleObjectError("get netmap via connection pool", err) + } + + return netmapSnapshot, nil +} + // ResolverFrostFS represents virtual connection to the FrostFS network. // It implements resolver.FrostFS. type ResolverFrostFS struct { diff --git a/internal/service/frostfs/source.go b/internal/service/frostfs/source.go new file mode 100644 index 0000000..de6c681 --- /dev/null +++ b/internal/service/frostfs/source.go @@ -0,0 +1,69 @@ +package frostfs + +import ( + "context" + "fmt" + + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" + "go.uber.org/zap" +) + +type Source struct { + frostFS *FrostFS + netmapCache *cache.NetmapCache + bucketCache *cache.BucketCache + log *zap.Logger +} + +func NewSource(frostFS *FrostFS, netmapCache *cache.NetmapCache, bucketCache *cache.BucketCache, log *zap.Logger) *Source { + return &Source{ + frostFS: frostFS, + netmapCache: netmapCache, + bucketCache: bucketCache, + log: log, + } +} + +func (s *Source) NetMapSnapshot(ctx context.Context) (netmap.NetMap, error) { + cachedNetmap := s.netmapCache.Get() + if cachedNetmap != nil { + return *cachedNetmap, nil + } + + netmapSnapshot, err := s.frostFS.NetmapSnapshot(ctx) + if err != nil { + return netmap.NetMap{}, fmt.Errorf("get netmap: %w", err) + } + + if err = s.netmapCache.Put(netmapSnapshot); err != nil { + s.log.Warn(logs.CouldntCacheNetmap, zap.Error(err)) + } + + return netmapSnapshot, nil +} + +func (s *Source) PlacementPolicy(ctx context.Context, cnrID cid.ID) (netmap.PlacementPolicy, error) { + info := s.bucketCache.GetByCID(cnrID) + if info != nil { + return info.PlacementPolicy, nil + } + + prm := handler.PrmContainer{ + ContainerID: cnrID, + } + res, err := s.frostFS.Container(ctx, prm) + if err != nil { + return netmap.PlacementPolicy{}, fmt.Errorf("get container: %w", err) + } + + // We don't put container back to the cache to keep cache + // coherent to the requests made by users. FrostFS Source + // is being used by SDK Tree Pool and it should not fill cache + // with possibly irrelevant container values. + + return res.PlacementPolicy(), nil +}