diff --git a/CHANGELOG.md b/CHANGELOG.md index e528b8b..fd37815 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ This document outlines major changes between releases. ## [Unreleased] +### Added +- Add handling quota limit reached error (#187) + ## [0.32.0] - Khumbu - 2024-12-20 ### Fixed diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index 3386536..e34386c 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -25,7 +25,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/templates" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/metrics" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" @@ -636,28 +635,28 @@ func (a *app) stopServices() { } } -func (a *app) configureRouter(handler *handler.Handler) { +func (a *app) configureRouter(h *handler.Handler) { r := router.New() r.RedirectTrailingSlash = true r.NotFound = func(r *fasthttp.RequestCtx) { - response.Error(r, "Not found", fasthttp.StatusNotFound) + handler.ResponseError(r, "Not found", fasthttp.StatusNotFound) } r.MethodNotAllowed = func(r *fasthttp.RequestCtx) { - response.Error(r, "Method Not Allowed", fasthttp.StatusMethodNotAllowed) + handler.ResponseError(r, "Method Not Allowed", fasthttp.StatusMethodNotAllowed) } - r.POST("/upload/{cid}", a.addMiddlewares(handler.Upload)) + r.POST("/upload/{cid}", a.addMiddlewares(h.Upload)) r.OPTIONS("/upload/{cid}", a.addPreflight()) a.log.Info(logs.AddedPathUploadCid) - r.GET("/get/{cid}/{oid:*}", a.addMiddlewares(handler.DownloadByAddressOrBucketName)) - r.HEAD("/get/{cid}/{oid:*}", a.addMiddlewares(handler.HeadByAddressOrBucketName)) + r.GET("/get/{cid}/{oid:*}", a.addMiddlewares(h.DownloadByAddressOrBucketName)) + r.HEAD("/get/{cid}/{oid:*}", a.addMiddlewares(h.HeadByAddressOrBucketName)) r.OPTIONS("/get/{cid}/{oid:*}", a.addPreflight()) a.log.Info(logs.AddedPathGetCidOid) - r.GET("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.addMiddlewares(handler.DownloadByAttribute)) - r.HEAD("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.addMiddlewares(handler.HeadByAttribute)) + r.GET("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.addMiddlewares(h.DownloadByAttribute)) + r.HEAD("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.addMiddlewares(h.HeadByAttribute)) r.OPTIONS("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.addPreflight()) a.log.Info(logs.AddedPathGetByAttributeCidAttrKeyAttrVal) - r.GET("/zip/{cid}/{prefix:*}", a.addMiddlewares(handler.DownloadZipped)) + r.GET("/zip/{cid}/{prefix:*}", a.addMiddlewares(h.DownloadZipped)) r.OPTIONS("/zip/{cid}/{prefix:*}", a.addPreflight()) a.log.Info(logs.AddedPathZipCidPrefix) @@ -800,7 +799,7 @@ func (a *app) tokenizer(h fasthttp.RequestHandler) fasthttp.RequestHandler { log := utils.GetReqLogOrDefault(reqCtx, a.log) log.Error(logs.CouldNotFetchAndStoreBearerToken, zap.Error(err)) - response.Error(req, "could not fetch and store bearer token: "+err.Error(), fasthttp.StatusBadRequest) + handler.ResponseError(req, "could not fetch and store bearer token: "+err.Error(), fasthttp.StatusBadRequest) return } utils.SetContextToRequest(appCtx, req) diff --git a/internal/handler/download.go b/internal/handler/download.go index de27fa3..8766f0c 100644 --- a/internal/handler/download.go +++ b/internal/handler/download.go @@ -13,7 +13,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -120,7 +119,7 @@ func (h *Handler) DownloadZipped(c *fasthttp.RequestCtx) { prefix, err := url.QueryUnescape(prefix) if err != nil { log.Error(logs.FailedToUnescapeQuery, zap.String("cid", scid), zap.String("prefix", prefix), zap.Error(err)) - response.Error(c, "could not unescape prefix: "+err.Error(), fasthttp.StatusBadRequest) + ResponseError(c, "could not unescape prefix: "+err.Error(), fasthttp.StatusBadRequest) return } @@ -135,7 +134,7 @@ func (h *Handler) DownloadZipped(c *fasthttp.RequestCtx) { resSearch, err := h.search(ctx, bktInfo.CID, object.AttributeFilePath, prefix, object.MatchCommonPrefix) if err != nil { log.Error(logs.CouldNotSearchForObjects, zap.Error(err)) - response.Error(c, "could not search for objects: "+err.Error(), fasthttp.StatusBadRequest) + ResponseError(c, "could not search for objects: "+err.Error(), fasthttp.StatusBadRequest) return } diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 1150f45..2f1c6ad 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -13,7 +13,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -140,6 +139,8 @@ var ( ErrAccessDenied = errors.New("access denied") // ErrGatewayTimeout is returned from FrostFS in case of timeout, deadline exceeded etc. ErrGatewayTimeout = errors.New("gateway timeout") + // ErrQuotaLimitReached is returned from FrostFS in case of quota exceeded. + ErrQuotaLimitReached = errors.New("quota limit reached") ) // FrostFS represents virtual connection to FrostFS network. @@ -210,7 +211,7 @@ func (h *Handler) byS3Path(ctx context.Context, req request, cnrID cid.ID, path } if foundOID.IsDeleteMarker { log.Error(logs.ObjectWasDeleted) - response.Error(c, "object deleted", fasthttp.StatusNotFound) + ResponseError(c, "object deleted", fasthttp.StatusNotFound) return } @@ -230,14 +231,14 @@ func (h *Handler) byAttribute(c *fasthttp.RequestCtx, handler func(context.Conte key, err := url.QueryUnescape(key) if err != nil { log.Error(logs.FailedToUnescapeQuery, zap.String("cid", cidParam), zap.String("attr_key", key), zap.Error(err)) - response.Error(c, "could not unescape attr_key: "+err.Error(), fasthttp.StatusBadRequest) + ResponseError(c, "could not unescape attr_key: "+err.Error(), fasthttp.StatusBadRequest) return } val, err = url.QueryUnescape(val) if err != nil { log.Error(logs.FailedToUnescapeQuery, zap.String("cid", cidParam), zap.String("attr_val", val), zap.Error(err)) - response.Error(c, "could not unescape attr_val: "+err.Error(), fasthttp.StatusBadRequest) + ResponseError(c, "could not unescape attr_val: "+err.Error(), fasthttp.StatusBadRequest) return } @@ -252,11 +253,11 @@ func (h *Handler) byAttribute(c *fasthttp.RequestCtx, handler func(context.Conte objID, err := h.findObjectByAttribute(ctx, log, bktInfo.CID, key, val) if err != nil { if errors.Is(err, io.EOF) { - response.Error(c, err.Error(), fasthttp.StatusNotFound) + ResponseError(c, err.Error(), fasthttp.StatusNotFound) return } - response.Error(c, err.Error(), fasthttp.StatusBadRequest) + ResponseError(c, err.Error(), fasthttp.StatusBadRequest) return } diff --git a/internal/handler/reader.go b/internal/handler/reader.go index 60067ab..cbd8294 100644 --- a/internal/handler/reader.go +++ b/internal/handler/reader.go @@ -12,7 +12,6 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -146,7 +145,7 @@ func (h *Handler) receiveFile(ctx context.Context, req request, objAddress oid.A }, filename) if err != nil && err != io.EOF { req.log.Error(logs.CouldNotDetectContentTypeFromPayload, zap.Error(err)) - response.Error(req.RequestCtx, "could not detect Content-Type from payload: "+err.Error(), fasthttp.StatusBadRequest) + ResponseError(req.RequestCtx, "could not detect Content-Type from payload: "+err.Error(), fasthttp.StatusBadRequest) return } diff --git a/internal/handler/upload.go b/internal/handler/upload.go index 867025d..9493635 100644 --- a/internal/handler/upload.go +++ b/internal/handler/upload.go @@ -9,7 +9,6 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" @@ -81,14 +80,14 @@ func (h *Handler) Upload(c *fasthttp.RequestCtx) { boundary := string(c.Request.Header.MultipartFormBoundary()) if file, err = fetchMultipartFile(log, bodyStream, boundary); err != nil { log.Error(logs.CouldNotReceiveMultipartForm, zap.Error(err)) - response.Error(c, "could not receive multipart/form: "+err.Error(), fasthttp.StatusBadRequest) + ResponseError(c, "could not receive multipart/form: "+err.Error(), fasthttp.StatusBadRequest) return } filtered, err := filterHeaders(log, &c.Request.Header) if err != nil { log.Error(logs.CouldNotProcessHeaders, zap.Error(err)) - response.Error(c, err.Error(), fasthttp.StatusBadRequest) + ResponseError(c, err.Error(), fasthttp.StatusBadRequest) return } @@ -103,7 +102,7 @@ func (h *Handler) Upload(c *fasthttp.RequestCtx) { if err = utils.PrepareExpirationHeader(c, h.frostfs, filtered, now); err != nil { log.Error(logs.CouldNotPrepareExpirationHeader, zap.Error(err)) - response.Error(c, "could not prepare expiration header: "+err.Error(), fasthttp.StatusBadRequest) + ResponseError(c, "could not prepare expiration header: "+err.Error(), fasthttp.StatusBadRequest) return } @@ -157,7 +156,7 @@ func (h *Handler) Upload(c *fasthttp.RequestCtx) { // Try to return the response, otherwise, if something went wrong, throw an error. if err = newPutResponse(addr).encode(c); err != nil { log.Error(logs.CouldNotEncodeResponse, zap.Error(err)) - response.Error(c, "could not encode response", fasthttp.StatusBadRequest) + ResponseError(c, "could not encode response", fasthttp.StatusBadRequest) return } @@ -179,11 +178,11 @@ func (h *Handler) Upload(c *fasthttp.RequestCtx) { } func (h *Handler) handlePutFrostFSErr(r *fasthttp.RequestCtx, err error, log *zap.Logger) { - statusCode, msg, additionalFields := response.FormErrorResponse("could not store file in frostfs", err) + statusCode, msg, additionalFields := formErrorResponse("could not store file in frostfs", err) logFields := append([]zap.Field{zap.Error(err)}, additionalFields...) log.Error(logs.CouldNotStoreFileInFrostfs, logFields...) - response.Error(r, msg, statusCode) + ResponseError(r, msg, statusCode) } func (h *Handler) fetchBearerToken(ctx context.Context) *bearer.Token { diff --git a/internal/handler/utils.go b/internal/handler/utils.go index d09ed23..7fdd396 100644 --- a/internal/handler/utils.go +++ b/internal/handler/utils.go @@ -2,14 +2,16 @@ package handler import ( "context" + "errors" + "fmt" "strings" "time" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" + sdkstatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -27,11 +29,11 @@ func (r *request) handleFrostFSErr(err error, start time.Time) { zap.Stringer("elapsed", time.Since(start)), zap.Error(err), } - statusCode, msg, additionalFields := response.FormErrorResponse("could not receive object", err) + statusCode, msg, additionalFields := formErrorResponse("could not receive object", err) logFields = append(logFields, additionalFields...) r.log.Error(logs.CouldNotReceiveObject, logFields...) - response.Error(r.RequestCtx, msg, statusCode) + ResponseError(r.RequestCtx, msg, statusCode) } func bearerToken(ctx context.Context) *bearer.Token { @@ -79,10 +81,10 @@ func logAndSendBucketError(c *fasthttp.RequestCtx, log *zap.Logger, err error) { log.Error(logs.CouldntGetBucket, zap.Error(err)) if client.IsErrContainerNotFound(err) { - response.Error(c, "Not Found", fasthttp.StatusNotFound) + ResponseError(c, "Not Found", fasthttp.StatusNotFound) return } - response.Error(c, "could not get bucket: "+err.Error(), fasthttp.StatusBadRequest) + ResponseError(c, "could not get bucket: "+err.Error(), fasthttp.StatusBadRequest) } func newAddress(cnr cid.ID, obj oid.ID) oid.Address { @@ -91,3 +93,36 @@ func newAddress(cnr cid.ID, obj oid.ID) oid.Address { addr.SetObject(obj) return addr } + +func ResponseError(r *fasthttp.RequestCtx, msg string, code int) { + r.Error(msg+"\n", code) +} + +func formErrorResponse(message string, err error) (int, string, []zap.Field) { + var ( + msg string + statusCode int + logFields []zap.Field + ) + + st := new(sdkstatus.ObjectAccessDenied) + + switch { + case errors.As(err, &st): + statusCode = fasthttp.StatusForbidden + reason := st.Reason() + msg = fmt.Sprintf("%s: %v: %s", message, err, reason) + logFields = append(logFields, zap.String("error_detail", reason)) + case errors.Is(err, ErrQuotaLimitReached): + statusCode = fasthttp.StatusConflict + msg = fmt.Sprintf("%s: %v", message, err) + case client.IsErrObjectNotFound(err) || client.IsErrContainerNotFound(err): + statusCode = fasthttp.StatusNotFound + msg = "Not Found" + default: + statusCode = fasthttp.StatusBadRequest + msg = fmt.Sprintf("%s: %v", message, err) + } + + return statusCode, msg, logFields +} diff --git a/internal/service/frostfs/frostfs.go b/internal/service/frostfs/frostfs.go index b218976..c6af526 100644 --- a/internal/service/frostfs/frostfs.go +++ b/internal/service/frostfs/frostfs.go @@ -215,6 +215,10 @@ func handleObjectError(msg string, err error) error { } if reason, ok := IsErrObjectAccessDenied(err); ok { + if strings.Contains(reason, "limit reached") { + return fmt.Errorf("%s: %w: %s", msg, handler.ErrQuotaLimitReached, reason) + } + return fmt.Errorf("%s: %w: %s", msg, handler.ErrAccessDenied, reason) } diff --git a/internal/service/frostfs/frostfs_test.go b/internal/service/frostfs/frostfs_test.go new file mode 100644 index 0000000..e9b3329 --- /dev/null +++ b/internal/service/frostfs/frostfs_test.go @@ -0,0 +1,83 @@ +package frostfs + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func TestHandleObjectError(t *testing.T) { + msg := "some msg" + + t.Run("nil error", func(t *testing.T) { + err := handleObjectError(msg, nil) + require.Nil(t, err) + }) + + t.Run("simple access denied", func(t *testing.T) { + reason := "some reason" + inputErr := new(apistatus.ObjectAccessDenied) + inputErr.WriteReason(reason) + + err := handleObjectError(msg, inputErr) + require.ErrorIs(t, err, handler.ErrAccessDenied) + require.Contains(t, err.Error(), reason) + require.Contains(t, err.Error(), msg) + }) + + t.Run("access denied - quota reached", func(t *testing.T) { + reason := "Quota limit reached" + inputErr := new(apistatus.ObjectAccessDenied) + inputErr.WriteReason(reason) + + err := handleObjectError(msg, inputErr) + require.ErrorIs(t, err, handler.ErrQuotaLimitReached) + require.Contains(t, err.Error(), reason) + require.Contains(t, err.Error(), msg) + }) + + t.Run("simple timeout", func(t *testing.T) { + inputErr := errors.New("timeout") + + err := handleObjectError(msg, inputErr) + require.ErrorIs(t, err, handler.ErrGatewayTimeout) + require.Contains(t, err.Error(), inputErr.Error()) + require.Contains(t, err.Error(), msg) + }) + + t.Run("deadline exceeded", func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) + defer cancel() + <-ctx.Done() + + err := handleObjectError(msg, ctx.Err()) + require.ErrorIs(t, err, handler.ErrGatewayTimeout) + require.Contains(t, err.Error(), ctx.Err().Error()) + require.Contains(t, err.Error(), msg) + }) + + t.Run("grpc deadline exceeded", func(t *testing.T) { + inputErr := fmt.Errorf("wrap grpc error: %w", status.Error(codes.DeadlineExceeded, "error")) + + err := handleObjectError(msg, inputErr) + require.ErrorIs(t, err, handler.ErrGatewayTimeout) + require.Contains(t, err.Error(), inputErr.Error()) + require.Contains(t, err.Error(), msg) + }) + + t.Run("unknown error", func(t *testing.T) { + inputErr := errors.New("unknown error") + + err := handleObjectError(msg, inputErr) + require.ErrorIs(t, err, inputErr) + require.Contains(t, err.Error(), msg) + }) +} diff --git a/response/utils.go b/response/utils.go deleted file mode 100644 index f233943..0000000 --- a/response/utils.go +++ /dev/null @@ -1,41 +0,0 @@ -package response - -import ( - "errors" - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" - sdkstatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" - "github.com/valyala/fasthttp" - "go.uber.org/zap" -) - -func Error(r *fasthttp.RequestCtx, msg string, code int) { - r.Error(msg+"\n", code) -} - -func FormErrorResponse(message string, err error) (int, string, []zap.Field) { - var ( - msg string - statusCode int - logFields []zap.Field - ) - - st := new(sdkstatus.ObjectAccessDenied) - - switch { - case errors.As(err, &st): - statusCode = fasthttp.StatusForbidden - reason := st.Reason() - msg = fmt.Sprintf("%s: %v: %s", message, err, reason) - logFields = append(logFields, zap.String("error_detail", reason)) - case client.IsErrObjectNotFound(err) || client.IsErrContainerNotFound(err): - statusCode = fasthttp.StatusNotFound - msg = "Not Found" - default: - statusCode = fasthttp.StatusBadRequest - msg = fmt.Sprintf("%s: %v", message, err) - } - - return statusCode, msg, logFields -}