From 9a5a2239bd0ef00e32f68dcb4d478293cb67dedf Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Wed, 4 Oct 2023 14:50:37 +0300 Subject: [PATCH] [#70] Support bucket/container caching Mainly it was added because we need to know if TZ hashing is disabled or not for container Signed-off-by: Denis Kirillov --- CHANGELOG.md | 6 ++- cmd/http-gw/app.go | 2 + cmd/http-gw/settings.go | 46 ++++++++++++++++++ config/config.env | 5 ++ config/config.yaml | 7 +++ docs/gate-configuration.md | 29 ++++++++++++ go.mod | 1 + go.sum | 2 + internal/cache/buckets.go | 68 +++++++++++++++++++++++++++ internal/data/bucket.go | 12 +++++ internal/handler/download.go | 34 ++------------ internal/handler/handler.go | 90 +++++++++++++++++++++++++++++++----- internal/handler/head.go | 2 +- internal/handler/upload.go | 10 ++-- internal/handler/utils.go | 11 +++++ internal/logs/logs.go | 4 +- utils/params.go | 2 + 17 files changed, 283 insertions(+), 48 deletions(-) create mode 100644 internal/cache/buckets.go create mode 100644 internal/data/bucket.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 10d42c0..4618d35 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,11 @@ This document outlines major changes between releases. - Support impersonate bearer token (#40, #45) - Tracing support (#20, #44, #60) - Object name resolving with tree service (#30) -- Add new `frostfs.client_cut` and `frostfs.buffer_max_size_for_put` config params (#70) +- Support client side object cut (#70) + - Add `frostfs.client_cut` config param + - Add `frostfs.buffer_max_size_for_put` config param + - Add bucket/container caching + - Disable homomorphic hash for PUT if it's disabled in container itself ### Changed - Update prometheus to v1.15.0 (#35) diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index 81bfb14..3878277 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -11,6 +11,7 @@ import ( "syscall" "time" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/frostfs/services" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" @@ -567,6 +568,7 @@ func (a *app) AppParams() *utils.AppParams { Pool: a.pool, Owner: a.owner, Resolver: a.resolver, + Cache: cache.NewBucketCache(getCacheOptions(a.cfg, a.log)), } } diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index b097aa3..cb309b7 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -13,6 +13,7 @@ import ( "strings" "time" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" grpctracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc" @@ -103,6 +104,10 @@ const ( // Sets max buffer size for read payload in put operations. cfgBufferMaxSizeForPut = "frostfs.buffer_max_size_for_put" + // Caching. + cfgBucketsCacheLifetime = "cache.buckets.lifetime" + cfgBucketsCacheSize = "cache.buckets.size" + // Command line args. cmdHelp = "help" cmdVersion = "version" @@ -541,3 +546,44 @@ func fetchSoftMemoryLimit(cfg *viper.Viper) int64 { return int64(softMemoryLimit) } + +func getCacheOptions(v *viper.Viper, l *zap.Logger) *cache.Config { + cacheCfg := cache.DefaultBucketConfig(l) + + cacheCfg.Lifetime = fetchCacheLifetime(v, l, cfgBucketsCacheLifetime, cacheCfg.Lifetime) + cacheCfg.Size = fetchCacheSize(v, l, cfgBucketsCacheSize, cacheCfg.Size) + + return cacheCfg +} + +func fetchCacheLifetime(v *viper.Viper, l *zap.Logger, cfgEntry string, defaultValue time.Duration) time.Duration { + if v.IsSet(cfgEntry) { + lifetime := v.GetDuration(cfgEntry) + if lifetime <= 0 { + l.Error("invalid lifetime, using default value (in seconds)", + zap.String("parameter", cfgEntry), + zap.Duration("value in config", lifetime), + zap.Duration("default", defaultValue)) + } else { + return lifetime + } + } + + return defaultValue +} + +func fetchCacheSize(v *viper.Viper, l *zap.Logger, cfgEntry string, defaultValue int) int { + if v.IsSet(cfgEntry) { + size := v.GetInt(cfgEntry) + if size <= 0 { + l.Error("invalid cache size, using default value", + zap.String("parameter", cfgEntry), + zap.Int("value in config", size), + zap.Int("default", defaultValue)) + } else { + return size + } + } + + return defaultValue +} diff --git a/config/config.env b/config/config.env index 06e71f8..739cb96 100644 --- a/config/config.env +++ b/config/config.env @@ -104,3 +104,8 @@ HTTP_GW_RUNTIME_SOFT_MEMORY_LIMIT=1073741824 HTTP_GW_FROSTFS_CLIENT_CUT=false # Sets max buffer size for read payload in put operations. HTTP_GW_FROSTFS_BUFFER_MAX_SIZE_FOR_PUT=1048576 + +# Caching +# Cache which contains mapping of bucket name to bucket info +HTTP_GW_CACHE_BUCKETS_LIFETIME=1m +HTTP_GW_CACHE_BUCKETS_SIZE=1000 diff --git a/config/config.yaml b/config/config.yaml index 15afeec..2cd20b5 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -111,3 +111,10 @@ frostfs: client_cut: false # Sets max buffer size for read payload in put operations. buffer_max_size_for_put: 1048576 + +# Caching +cache: + # Cache which contains mapping of bucket name to bucket info + buckets: + lifetime: 1m + size: 1000 diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index 852908b..65fe618 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -55,6 +55,7 @@ $ cat http.log | `tracing` | [Tracing configuration](#tracing-section) | | `runtime` | [Runtime configuration](#runtime-section) | | `frostfs` | [Frostfs configuration](#frostfs-section) | +| `cache` | [Cache configuration](#cache-section) | # General section @@ -285,3 +286,31 @@ frostfs: |---------------------------|----------|---------------|---------------|----------------------------------------------------------| | `client_cut` | `bool` | yes | `false` | This flag enables client side object preparing. | | `buffer_max_size_for_put` | `uint64` | yes | `1048576` | Sets max buffer size for read payload in put operations. | + + +### `cache` section + +```yaml +cache: + buckets: + lifetime: 1m + size: 1000 + +``` + +| Parameter | Type | Default value | Description | +|-----------------|-----------------------------------|-----------------------------------|----------------------------------------------------------------------------------------| +| `buckets` | [Cache config](#cache-subsection) | `lifetime: 60s`
`size: 1000` | Cache which contains mapping of bucket name to bucket info. | + + +#### `cache` subsection + +```yaml +lifetime: 1m +size: 1000 +``` + +| Parameter | Type | Default value | Description | +|------------|------------|------------------|-------------------------------| +| `lifetime` | `duration` | depends on cache | Lifetime of entries in cache. | +| `size` | `int` | depends on cache | LRU cache size. | diff --git a/go.mod b/go.mod index a58272e..5f9b1b0 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230802075510-964c3edb3f44 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230825064515-46a214d065f8 + github.com/bluele/gcache v0.0.2 github.com/fasthttp/router v1.4.1 github.com/nspcc-dev/neo-go v0.101.2-0.20230601131642-a0117042e8fc github.com/prometheus/client_golang v1.15.1 diff --git a/go.sum b/go.sum index 0b9b457..dedb570 100644 --- a/go.sum +++ b/go.sum @@ -138,6 +138,8 @@ github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngE github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw= +github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/bshuster-repo/logrus-logstash-hook v0.4.1/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk= github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= diff --git a/internal/cache/buckets.go b/internal/cache/buckets.go new file mode 100644 index 0000000..abeda6a --- /dev/null +++ b/internal/cache/buckets.go @@ -0,0 +1,68 @@ +package cache + +import ( + "fmt" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" + "github.com/bluele/gcache" + "go.uber.org/zap" +) + +// BucketCache contains cache with objects and the lifetime of cache entries. +type BucketCache struct { + cache gcache.Cache + logger *zap.Logger +} + +// Config stores expiration params for cache. +type Config struct { + Size int + Lifetime time.Duration + Logger *zap.Logger +} + +const ( + // DefaultBucketCacheSize is a default maximum number of entries in cache. + DefaultBucketCacheSize = 1e3 + // DefaultBucketCacheLifetime is a default lifetime of entries in cache. + DefaultBucketCacheLifetime = time.Minute +) + +// DefaultBucketConfig returns new default cache expiration values. +func DefaultBucketConfig(logger *zap.Logger) *Config { + return &Config{ + Size: DefaultBucketCacheSize, + Lifetime: DefaultBucketCacheLifetime, + Logger: logger, + } +} + +// NewBucketCache creates an object of BucketCache. +func NewBucketCache(config *Config) *BucketCache { + gc := gcache.New(config.Size).LRU().Expiration(config.Lifetime).Build() + return &BucketCache{cache: gc, logger: config.Logger} +} + +// Get returns a cached object. +func (o *BucketCache) Get(key string) *data.BucketInfo { + entry, err := o.cache.Get(key) + if err != nil { + return nil + } + + result, ok := entry.(*data.BucketInfo) + if !ok { + o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)), + zap.String("expected", fmt.Sprintf("%T", result))) + return nil + } + + return result +} + +// Put puts an object to cache. +func (o *BucketCache) Put(bkt *data.BucketInfo) error { + return o.cache.Set(bkt.Name, bkt) +} diff --git a/internal/data/bucket.go b/internal/data/bucket.go new file mode 100644 index 0000000..d99ca49 --- /dev/null +++ b/internal/data/bucket.go @@ -0,0 +1,12 @@ +package data + +import ( + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" +) + +type BucketInfo struct { + Name string // container name from system attribute + Zone string // container zone from system attribute + CID cid.ID + HomomorphicHashDisabled bool +} diff --git a/internal/handler/download.go b/internal/handler/download.go index 5021b4a..06a247a 100644 --- a/internal/handler/download.go +++ b/internal/handler/download.go @@ -14,8 +14,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -30,7 +28,7 @@ func (h *Handler) DownloadByAddressOrBucketName(c *fasthttp.RequestCtx) { var id oid.ID err := id.DecodeString(test) if err != nil { - h.byBucketname(c, h.receiveFile) + h.byObjectName(c, h.receiveFile) } else { h.byAddress(c, h.receiveFile) } @@ -63,14 +61,6 @@ func (h *Handler) search(ctx context.Context, cid *cid.ID, key, val string, op o return h.pool.SearchObjects(ctx, prm) } -func (h *Handler) getContainer(ctx context.Context, cnrID cid.ID) (container.Container, error) { - prm := pool.PrmContainerGet{ - ContainerID: cnrID, - } - - return h.pool.GetContainer(ctx, prm) -} - func (h *Handler) addObjectToZip(zw *zip.Writer, obj *object.Object) (io.Writer, error) { method := zip.Store if h.config.ZipCompression() { @@ -97,27 +87,13 @@ func (h *Handler) DownloadZipped(c *fasthttp.RequestCtx) { ctx := utils.GetContextFromRequest(c) - containerID, err := h.getContainerID(ctx, scid) + bktInfo, err := h.getBucketInfo(ctx, scid, log) if err != nil { - log.Error(logs.WrongContainerID, zap.Error(err)) - response.Error(c, "wrong container id", fasthttp.StatusBadRequest) + logAndSendBucketError(c, log, err) return } - // check if container exists here to be able to return 404 error, - // otherwise we get this error only in object iteration step - // and client get 200 OK. - if _, err = h.getContainer(ctx, *containerID); err != nil { - log.Error(logs.CouldNotCheckContainerExistence, zap.Error(err)) - if client.IsErrContainerNotFound(err) { - response.Error(c, "Not Found", fasthttp.StatusNotFound) - return - } - response.Error(c, "could not check container existence: "+err.Error(), fasthttp.StatusBadRequest) - return - } - - resSearch, err := h.search(ctx, containerID, object.AttributeFilePath, prefix, object.MatchCommonPrefix) + resSearch, err := h.search(ctx, &bktInfo.CID, object.AttributeFilePath, prefix, object.MatchCommonPrefix) if err != nil { log.Error(logs.CouldNotSearchForObjects, zap.Error(err)) response.Error(c, "could not search for objects: "+err.Error(), fasthttp.StatusBadRequest) @@ -139,7 +115,7 @@ func (h *Handler) DownloadZipped(c *fasthttp.RequestCtx) { empty := true called := false btoken := bearerToken(ctx) - addr.SetContainer(*containerID) + addr.SetContainer(bktInfo.CID) errIter := resSearch.Iterate(func(id oid.ID) bool { called = true diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 61e2bc1..54602c2 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -3,14 +3,20 @@ package handler import ( "context" "errors" + "fmt" "io" "net/url" + "strings" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -34,6 +40,7 @@ type Handler struct { config Config containerResolver *resolver.ContainerResolver tree *tree.Tree + cache *cache.BucketCache } func New(params *utils.AppParams, config Config, tree *tree.Tree) *Handler { @@ -44,6 +51,7 @@ func New(params *utils.AppParams, config Config, tree *tree.Tree) *Handler { config: config, containerResolver: params.Resolver, tree: tree, + cache: params.Cache, } } @@ -69,10 +77,9 @@ func (h *Handler) byAddress(c *fasthttp.RequestCtx, f func(context.Context, requ ctx := utils.GetContextFromRequest(c) - cnrID, err := h.getContainerID(ctx, idCnr) + bktInfo, err := h.getBucketInfo(ctx, idCnr, log) if err != nil { - log.Error(logs.WrongContainerID, zap.Error(err)) - response.Error(c, "wrong container id", fasthttp.StatusBadRequest) + logAndSendBucketError(c, log, err) return } @@ -84,15 +91,15 @@ func (h *Handler) byAddress(c *fasthttp.RequestCtx, f func(context.Context, requ } var addr oid.Address - addr.SetContainer(*cnrID) + addr.SetContainer(bktInfo.CID) addr.SetObject(*objID) f(ctx, *h.newRequest(c, log), addr) } -// byBucketname is a wrapper for function (e.g. request.headObject, request.receiveFile) that +// byObjectName is a wrapper for function (e.g. request.headObject, request.receiveFile) that // prepares request and object address to it. -func (h *Handler) byBucketname(req *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) { +func (h *Handler) byObjectName(req *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) { var ( bucketname = req.UserValue("cid").(string) key = req.UserValue("oid").(string) @@ -101,14 +108,13 @@ func (h *Handler) byBucketname(req *fasthttp.RequestCtx, f func(context.Context, ctx := utils.GetContextFromRequest(req) - cnrID, err := h.getContainerID(ctx, bucketname) + bktInfo, err := h.getBucketInfo(ctx, bucketname, log) if err != nil { - log.Error(logs.WrongContainerID, zap.Error(err)) - response.Error(req, "wrong container id", fasthttp.StatusBadRequest) + logAndSendBucketError(req, log, err) return } - foundOid, err := h.tree.GetLatestVersion(ctx, cnrID, key) + foundOid, err := h.tree.GetLatestVersion(ctx, &bktInfo.CID, key) if err != nil { log.Error(logs.ObjectWasntFound, zap.Error(err)) response.Error(req, "object wasn't found", fasthttp.StatusNotFound) @@ -121,7 +127,7 @@ func (h *Handler) byBucketname(req *fasthttp.RequestCtx, f func(context.Context, } var addr oid.Address - addr.SetContainer(*cnrID) + addr.SetContainer(bktInfo.CID) addr.SetObject(foundOid.OID) f(ctx, *h.newRequest(req, log), addr) @@ -175,3 +181,65 @@ func (h *Handler) byAttribute(c *fasthttp.RequestCtx, f func(context.Context, re f(ctx, *h.newRequest(c, log), addrObj) } + +// resolveContainer decode container id, if it's not a valid container id +// then trey to resolve name using provided resolver. +func (h *Handler) resolveContainer(ctx context.Context, containerID string) (*cid.ID, error) { + cnrID := new(cid.ID) + err := cnrID.DecodeString(containerID) + if err != nil { + cnrID, err = h.containerResolver.Resolve(ctx, containerID) + if err != nil && strings.Contains(err.Error(), "not found") { + err = fmt.Errorf("%w: %s", &apistatus.ContainerNotFound{}, err.Error()) + + } + } + return cnrID, err +} + +func (h *Handler) getBucketInfo(ctx context.Context, containerName string, log *zap.Logger) (*data.BucketInfo, error) { + if bktInfo := h.cache.Get(containerName); bktInfo != nil { + return bktInfo, nil + } + + cnrID, err := h.resolveContainer(ctx, containerName) + if err != nil { + return nil, err + } + + bktInfo, err := h.readContainer(ctx, *cnrID) + if err != nil { + return nil, err + } + + if err = h.cache.Put(bktInfo); err != nil { + log.Warn(logs.CouldntPutBucketIntoCache, + zap.String("bucket name", bktInfo.Name), + zap.Stringer("bucket cid", bktInfo.CID), + zap.Error(err)) + } + + return bktInfo, nil +} + +func (h *Handler) readContainer(ctx context.Context, cnrID cid.ID) (*data.BucketInfo, error) { + prm := pool.PrmContainerGet{ContainerID: cnrID} + res, err := h.pool.GetContainer(ctx, prm) + if err != nil { + return nil, fmt.Errorf("get frostfs container '%s': %w", cnrID.String(), err) + } + + bktInfo := &data.BucketInfo{ + CID: cnrID, + Name: cnrID.EncodeToString(), + } + + if domain := container.ReadDomain(res); domain.Name() != "" { + bktInfo.Name = domain.Name() + bktInfo.Zone = domain.Zone() + } + + bktInfo.HomomorphicHashDisabled = container.IsHomomorphicHashingDisabled(res) + + return bktInfo, err +} diff --git a/internal/handler/head.go b/internal/handler/head.go index f7478f1..9418567 100644 --- a/internal/handler/head.go +++ b/internal/handler/head.go @@ -110,7 +110,7 @@ func (h *Handler) HeadByAddressOrBucketName(c *fasthttp.RequestCtx) { err := id.DecodeString(test) if err != nil { - h.byBucketname(c, h.headObject) + h.byObjectName(c, h.headObject) } else { h.byAddress(c, h.headObject) } diff --git a/internal/handler/upload.go b/internal/handler/upload.go index b95896f..935b51b 100644 --- a/internal/handler/upload.go +++ b/internal/handler/upload.go @@ -57,10 +57,9 @@ func (h *Handler) Upload(req *fasthttp.RequestCtx) { ctx := utils.GetContextFromRequest(req) - idCnr, err := h.getContainerID(ctx, scid) + bktInfo, err := h.getBucketInfo(ctx, scid, log) if err != nil { - log.Error(logs.WrongContainerID, zap.Error(err)) - response.Error(req, "wrong container id", fasthttp.StatusBadRequest) + logAndSendBucketError(req, log, err) return } @@ -129,7 +128,7 @@ func (h *Handler) Upload(req *fasthttp.RequestCtx) { } obj := object.New() - obj.SetContainerID(*idCnr) + obj.SetContainerID(bktInfo.CID) obj.SetOwnerID(h.ownerID) obj.SetAttributes(attributes...) @@ -138,6 +137,7 @@ func (h *Handler) Upload(req *fasthttp.RequestCtx) { prm.SetPayload(file) prm.SetClientCut(h.config.ClientCut()) prm.SetBufferMaxSize(h.config.BufferMaxSizeForPut()) + prm.WithoutHomomorphicHash(bktInfo.HomomorphicHashDisabled) bt := h.fetchBearerToken(ctx) if bt != nil { @@ -150,7 +150,7 @@ func (h *Handler) Upload(req *fasthttp.RequestCtx) { } addr.SetObject(idObj) - addr.SetContainer(*idCnr) + addr.SetContainer(bktInfo.CID) // Try to return the response, otherwise, if something went wrong, throw an error. if err = newPutResponse(addr).encode(req); err != nil { diff --git a/internal/handler/utils.go b/internal/handler/utils.go index b51400c..a5a53ed 100644 --- a/internal/handler/utils.go +++ b/internal/handler/utils.go @@ -9,6 +9,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" "github.com/valyala/fasthttp" "go.uber.org/zap" ) @@ -58,3 +59,13 @@ func isValidValue(s string) bool { } return true } + +func logAndSendBucketError(c *fasthttp.RequestCtx, log *zap.Logger, err error) { + log.Error(logs.CouldntGetBucket, zap.Error(err)) + + if client.IsErrContainerNotFound(err) { + response.Error(c, "Not Found", fasthttp.StatusNotFound) + return + } + response.Error(c, "could not get bucket: "+err.Error(), fasthttp.StatusBadRequest) +} diff --git a/internal/logs/logs.go b/internal/logs/logs.go index ebb3c24..79ddce5 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -11,7 +11,6 @@ const ( CouldNotSearchForObjects = "could not search for objects" // Error in ../../downloader/download.go ObjectNotFound = "object not found" // Error in ../../downloader/download.go ReadObjectListFailed = "read object list failed" // Error in ../../downloader/download.go - CouldNotCheckContainerExistence = "could not check container existence" // Error in ../../downloader/download.go FailedToAddObjectToArchive = "failed to add object to archive" // Error in ../../downloader/download.go IteratingOverSelectedObjectsFailed = "iterating over selected objects failed" // Error in ../../downloader/download.go ObjectsNotFound = "objects not found" // Error in ../../downloader/download.go @@ -68,4 +67,7 @@ const ( FailedToCreateTreePool = "failed to create tree pool" // Fatal in ../../settings.go FailedToDialTreePool = "failed to dial tree pool" // Fatal in ../../settings.go AddedStoragePeer = "added storage peer" // Info in ../../settings.go + CouldntGetBucket = "could not get bucket" // Error in ../handler/utils.go + CouldntPutBucketIntoCache = "couldn't put bucket info into cache" // Warn in ../handler/handler.go + InvalidCacheEntryType = "invalid cache entry type" // Warn in ../cache/buckets.go ) diff --git a/utils/params.go b/utils/params.go index a6fe59b..f27ff71 100644 --- a/utils/params.go +++ b/utils/params.go @@ -1,6 +1,7 @@ package utils import ( + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" @@ -12,4 +13,5 @@ type AppParams struct { Pool *pool.Pool Owner *user.ID Resolver *resolver.ContainerResolver + Cache *cache.BucketCache }