[#73] Uploader, downloader structures refactoring
All checks were successful
/ Vulncheck (pull_request) Successful in 1m32s
/ Lint (pull_request) Successful in 2m28s
/ Tests (1.20) (pull_request) Successful in 1m52s
/ Tests (1.21) (pull_request) Successful in 1m21s
/ DCO (pull_request) Successful in 3m25s
/ Builds (1.20) (pull_request) Successful in 5m38s
/ Builds (1.21) (pull_request) Successful in 1m23s

Signed-off-by: Marina Biryukova <m.biryukova@yadro.com>
This commit is contained in:
Marina Biryukova 2023-08-31 11:37:03 +03:00
parent add07a21ed
commit d219943542
27 changed files with 672 additions and 664 deletions

View file

@ -0,0 +1,210 @@
package handler
import (
"archive/zip"
"bufio"
"context"
"fmt"
"io"
"net/http"
"net/url"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/response"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
"github.com/valyala/fasthttp"
"go.uber.org/zap"
)
// DownloadByAddressOrBucketName handles download requests using simple cid/oid or bucketname/key format.
func (h *Handler) DownloadByAddressOrBucketName(c *fasthttp.RequestCtx) {
test, _ := c.UserValue("oid").(string)
var id oid.ID
err := id.DecodeString(test)
if err != nil {
h.byBucketname(c, h.receiveFile)
} else {
h.byAddress(c, h.receiveFile)
}
}
func (h *Handler) newRequest(ctx *fasthttp.RequestCtx, log *zap.Logger) *request {
return &request{
RequestCtx: ctx,
log: log,
}
}
// DownloadByAttribute handles attribute-based download requests.
func (h *Handler) DownloadByAttribute(c *fasthttp.RequestCtx) {
h.byAttribute(c, h.receiveFile)
}
func (h *Handler) search(ctx context.Context, cid *cid.ID, key, val string, op object.SearchMatchType) (pool.ResObjectSearch, error) {
filters := object.NewSearchFilters()
filters.AddRootFilter()
filters.AddFilter(key, val, op)
var prm pool.PrmObjectSearch
prm.SetContainerID(*cid)
prm.SetFilters(filters)
if btoken := bearerToken(ctx); btoken != nil {
prm.UseBearer(*btoken)
}
return h.pool.SearchObjects(ctx, prm)
}
func (h *Handler) getContainer(ctx context.Context, cnrID cid.ID) (container.Container, error) {
var prm pool.PrmContainerGet
prm.SetContainerID(cnrID)
return h.pool.GetContainer(ctx, prm)
}
func (h *Handler) addObjectToZip(zw *zip.Writer, obj *object.Object) (io.Writer, error) {
method := zip.Store
if h.settings.ZipCompression() {
method = zip.Deflate
}
filePath := getZipFilePath(obj)
if len(filePath) == 0 || filePath[len(filePath)-1] == '/' {
return nil, fmt.Errorf("invalid filepath '%s'", filePath)
}
return zw.CreateHeader(&zip.FileHeader{
Name: filePath,
Method: method,
Modified: time.Now(),
})
}
// DownloadZipped handles zip by prefix requests.
func (h *Handler) DownloadZipped(c *fasthttp.RequestCtx) {
scid, _ := c.UserValue("cid").(string)
prefix, _ := url.QueryUnescape(c.UserValue("prefix").(string))
log := h.log.With(zap.String("cid", scid), zap.String("prefix", prefix))
ctx := utils.GetContextFromRequest(c)
containerID, err := h.getContainerID(ctx, scid)
if err != nil {
log.Error(logs.WrongContainerID, zap.Error(err))
response.Error(c, "wrong container id", fasthttp.StatusBadRequest)
return
}
// check if container exists here to be able to return 404 error,
// otherwise we get this error only in object iteration step
// and client get 200 OK.
if _, err = h.getContainer(ctx, *containerID); err != nil {
log.Error(logs.CouldNotCheckContainerExistence, zap.Error(err))
if client.IsErrContainerNotFound(err) {
response.Error(c, "Not Found", fasthttp.StatusNotFound)
return
}
response.Error(c, "could not check container existence: "+err.Error(), fasthttp.StatusBadRequest)
return
}
resSearch, err := h.search(ctx, containerID, object.AttributeFilePath, prefix, object.MatchCommonPrefix)
if err != nil {
log.Error(logs.CouldNotSearchForObjects, zap.Error(err))
response.Error(c, "could not search for objects: "+err.Error(), fasthttp.StatusBadRequest)
return
}
c.Response.Header.Set(fasthttp.HeaderContentType, "application/zip")
c.Response.Header.Set(fasthttp.HeaderContentDisposition, "attachment; filename=\"archive.zip\"")
c.Response.SetStatusCode(http.StatusOK)
c.SetBodyStreamWriter(func(w *bufio.Writer) {
defer resSearch.Close()
zipWriter := zip.NewWriter(w)
var bufZip []byte
var addr oid.Address
empty := true
called := false
btoken := bearerToken(ctx)
addr.SetContainer(*containerID)
errIter := resSearch.Iterate(func(id oid.ID) bool {
called = true
if empty {
bufZip = make([]byte, 3<<20) // the same as for upload
}
empty = false
addr.SetObject(id)
if err = h.zipObject(ctx, zipWriter, addr, btoken, bufZip); err != nil {
log.Error(logs.FailedToAddObjectToArchive, zap.String("oid", id.EncodeToString()), zap.Error(err))
}
return false
})
if errIter != nil {
log.Error(logs.IteratingOverSelectedObjectsFailed, zap.Error(errIter))
} else if !called {
log.Error(logs.ObjectsNotFound)
}
if err = zipWriter.Close(); err != nil {
log.Error(logs.CloseZipWriter, zap.Error(err))
}
})
}
func (h *Handler) zipObject(ctx context.Context, zipWriter *zip.Writer, addr oid.Address, btoken *bearer.Token, bufZip []byte) error {
var prm pool.PrmObjectGet
prm.SetAddress(addr)
if btoken != nil {
prm.UseBearer(*btoken)
}
resGet, err := h.pool.GetObject(ctx, prm)
if err != nil {
return fmt.Errorf("get FrostFS object: %v", err)
}
objWriter, err := h.addObjectToZip(zipWriter, &resGet.Header)
if err != nil {
return fmt.Errorf("zip create header: %v", err)
}
if _, err = io.CopyBuffer(objWriter, resGet.Payload, bufZip); err != nil {
return fmt.Errorf("copy object payload to zip file: %v", err)
}
if err = resGet.Payload.Close(); err != nil {
return fmt.Errorf("object body close error: %w", err)
}
if err = zipWriter.Flush(); err != nil {
return fmt.Errorf("flush zip writer: %v", err)
}
return nil
}
func getZipFilePath(obj *object.Object) string {
for _, attr := range obj.Attributes() {
if attr.Key() == object.AttributeFilePath {
return attr.Value()
}
}
return ""
}

View file

@ -0,0 +1,57 @@
package handler
import (
"bytes"
"fmt"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
"github.com/valyala/fasthttp"
"go.uber.org/zap"
)
func filterHeaders(l *zap.Logger, header *fasthttp.RequestHeader) (map[string]string, error) {
var err error
result := make(map[string]string)
prefix := []byte(utils.UserAttributeHeaderPrefix)
header.VisitAll(func(key, val []byte) {
// checks that the key and the val not empty
if len(key) == 0 || len(val) == 0 {
return
}
// checks that the key has attribute prefix
if !bytes.HasPrefix(key, prefix) {
return
}
// removing attribute prefix
clearKey := bytes.TrimPrefix(key, prefix)
clearKey = utils.TransformIfSystem(clearKey)
// checks that the attribute key is not empty
if len(clearKey) == 0 {
return
}
// check if key gets duplicated
// return error containing full key name (with prefix)
if _, ok := result[string(clearKey)]; ok {
err = fmt.Errorf("key duplication error: %s", string(key))
return
}
// make string representation of key / val
k, v := string(clearKey), string(val)
result[k] = v
l.Debug(logs.AddAttributeToResultObject,
zap.String("key", k),
zap.String("val", v))
})
return result, err
}

View file

@ -0,0 +1,53 @@
//go:build !integration
package handler
import (
"testing"
"github.com/stretchr/testify/require"
"github.com/valyala/fasthttp"
"go.uber.org/zap"
)
func TestFilter(t *testing.T) {
log := zap.NewNop()
t.Run("duplicate keys error", func(t *testing.T) {
req := &fasthttp.RequestHeader{}
req.DisableNormalizing()
req.Add("X-Attribute-DupKey", "first-value")
req.Add("X-Attribute-DupKey", "second-value")
_, err := filterHeaders(log, req)
require.Error(t, err)
})
t.Run("duplicate system keys error", func(t *testing.T) {
req := &fasthttp.RequestHeader{}
req.DisableNormalizing()
req.Add("X-Attribute-System-DupKey", "first-value")
req.Add("X-Attribute-System-DupKey", "second-value")
_, err := filterHeaders(log, req)
require.Error(t, err)
})
req := &fasthttp.RequestHeader{}
req.DisableNormalizing()
req.Set("X-Attribute-System-Expiration-Epoch1", "101")
req.Set("X-Attribute-SYSTEM-Expiration-Epoch2", "102")
req.Set("X-Attribute-system-Expiration-Epoch3", "103")
req.Set("X-Attribute-MyAttribute", "value")
expected := map[string]string{
"__SYSTEM__EXPIRATION_EPOCH1": "101",
"MyAttribute": "value",
"__SYSTEM__EXPIRATION_EPOCH3": "103",
"__SYSTEM__EXPIRATION_EPOCH2": "102",
}
result, err := filterHeaders(log, req)
require.NoError(t, err)
require.Equal(t, expected, result)
}

193
internal/handler/handler.go Normal file
View file

@ -0,0 +1,193 @@
package handler
import (
"context"
"errors"
"io"
"net/url"
"sync/atomic"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/response"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
"github.com/valyala/fasthttp"
"go.uber.org/zap"
)
type Handler struct {
log *zap.Logger
pool *pool.Pool
ownerID *user.ID
settings *Settings
containerResolver *resolver.ContainerResolver
tree *tree.Tree
}
// Settings stores reloading parameters, so it has to provide atomic getters and setters.
type Settings struct {
defaultTimestamp atomic.Bool
zipCompression atomic.Bool
}
func (s *Settings) DefaultTimestamp() bool {
return s.defaultTimestamp.Load()
}
func (s *Settings) SetDefaultTimestamp(val bool) {
s.defaultTimestamp.Store(val)
}
func (s *Settings) ZipCompression() bool {
return s.zipCompression.Load()
}
func (s *Settings) SetZipCompression(val bool) {
s.zipCompression.Store(val)
}
func New(params *utils.AppParams, settings *Settings, tree *tree.Tree) *Handler {
return &Handler{
log: params.Logger,
pool: params.Pool,
ownerID: params.Owner,
settings: settings,
containerResolver: params.Resolver,
tree: tree,
}
}
// getContainerID decode container id, if it's not a valid container id
// then trey to resolve name using provided resolver.
func (h *Handler) getContainerID(ctx context.Context, containerID string) (*cid.ID, error) {
cnrID := new(cid.ID)
err := cnrID.DecodeString(containerID)
if err != nil {
cnrID, err = h.containerResolver.Resolve(ctx, containerID)
}
return cnrID, err
}
// byAddress is a wrapper for function (e.g. request.headObject, request.receiveFile) that
// prepares request and object address to it.
func (h *Handler) byAddress(c *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) {
var (
idCnr, _ = c.UserValue("cid").(string)
idObj, _ = c.UserValue("oid").(string)
log = h.log.With(zap.String("cid", idCnr), zap.String("oid", idObj))
)
ctx := utils.GetContextFromRequest(c)
cnrID, err := h.getContainerID(ctx, idCnr)
if err != nil {
log.Error(logs.WrongContainerID, zap.Error(err))
response.Error(c, "wrong container id", fasthttp.StatusBadRequest)
return
}
objID := new(oid.ID)
if err = objID.DecodeString(idObj); err != nil {
log.Error(logs.WrongObjectID, zap.Error(err))
response.Error(c, "wrong object id", fasthttp.StatusBadRequest)
return
}
var addr oid.Address
addr.SetContainer(*cnrID)
addr.SetObject(*objID)
f(ctx, *h.newRequest(c, log), addr)
}
// byBucketname is a wrapper for function (e.g. request.headObject, request.receiveFile) that
// prepares request and object address to it.
func (h *Handler) byBucketname(req *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) {
var (
bucketname = req.UserValue("cid").(string)
key = req.UserValue("oid").(string)
log = h.log.With(zap.String("bucketname", bucketname), zap.String("key", key))
)
ctx := utils.GetContextFromRequest(req)
cnrID, err := h.getContainerID(ctx, bucketname)
if err != nil {
log.Error(logs.WrongContainerID, zap.Error(err))
response.Error(req, "wrong container id", fasthttp.StatusBadRequest)
return
}
foundOid, err := h.tree.GetLatestVersion(ctx, cnrID, key)
if err != nil {
log.Error(logs.ObjectWasntFound, zap.Error(err))
response.Error(req, "object wasn't found", fasthttp.StatusNotFound)
return
}
if foundOid.DeleteMarker {
log.Error(logs.ObjectWasDeleted)
response.Error(req, "object deleted", fasthttp.StatusNotFound)
return
}
var addr oid.Address
addr.SetContainer(*cnrID)
addr.SetObject(foundOid.OID)
f(ctx, *h.newRequest(req, log), addr)
}
// byAttribute is a wrapper similar to byAddress.
func (h *Handler) byAttribute(c *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) {
var (
scid, _ = c.UserValue("cid").(string)
key, _ = url.QueryUnescape(c.UserValue("attr_key").(string))
val, _ = url.QueryUnescape(c.UserValue("attr_val").(string))
log = h.log.With(zap.String("cid", scid), zap.String("attr_key", key), zap.String("attr_val", val))
)
ctx := utils.GetContextFromRequest(c)
containerID, err := h.getContainerID(ctx, scid)
if err != nil {
log.Error(logs.WrongContainerID, zap.Error(err))
response.Error(c, "wrong container id", fasthttp.StatusBadRequest)
return
}
res, err := h.search(ctx, containerID, key, val, object.MatchStringEqual)
if err != nil {
log.Error(logs.CouldNotSearchForObjects, zap.Error(err))
response.Error(c, "could not search for objects: "+err.Error(), fasthttp.StatusBadRequest)
return
}
defer res.Close()
buf := make([]oid.ID, 1)
n, err := res.Read(buf)
if n == 0 {
if errors.Is(err, io.EOF) {
log.Error(logs.ObjectNotFound, zap.Error(err))
response.Error(c, "object not found", fasthttp.StatusNotFound)
return
}
log.Error(logs.ReadObjectListFailed, zap.Error(err))
response.Error(c, "read object list failed: "+err.Error(), fasthttp.StatusBadRequest)
return
}
var addrObj oid.Address
addrObj.SetContainer(*containerID)
addrObj.SetObject(buf[0])
f(ctx, *h.newRequest(c, log), addrObj)
}

122
internal/handler/head.go Normal file
View file

@ -0,0 +1,122 @@
package handler
import (
"context"
"io"
"net/http"
"strconv"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
"github.com/valyala/fasthttp"
"go.uber.org/zap"
)
// max bytes needed to detect content type according to http.DetectContentType docs.
const sizeToDetectType = 512
const (
hdrObjectID = "X-Object-Id"
hdrOwnerID = "X-Owner-Id"
hdrContainerID = "X-Container-Id"
)
func (h *Handler) headObject(ctx context.Context, req request, objectAddress oid.Address) {
var start = time.Now()
btoken := bearerToken(ctx)
var prm pool.PrmObjectHead
prm.SetAddress(objectAddress)
if btoken != nil {
prm.UseBearer(*btoken)
}
obj, err := h.pool.HeadObject(ctx, prm)
if err != nil {
req.handleFrostFSErr(err, start)
return
}
req.Response.Header.Set(fasthttp.HeaderContentLength, strconv.FormatUint(obj.PayloadSize(), 10))
var contentType string
for _, attr := range obj.Attributes() {
key := attr.Key()
val := attr.Value()
if !isValidToken(key) || !isValidValue(val) {
continue
}
key = utils.BackwardTransformIfSystem(key)
req.Response.Header.Set(utils.UserAttributeHeaderPrefix+key, val)
switch key {
case object.AttributeTimestamp:
value, err := strconv.ParseInt(val, 10, 64)
if err != nil {
req.log.Info(logs.CouldntParseCreationDate,
zap.String("key", key),
zap.String("val", val),
zap.Error(err))
continue
}
req.Response.Header.Set(fasthttp.HeaderLastModified, time.Unix(value, 0).UTC().Format(http.TimeFormat))
case object.AttributeContentType:
contentType = val
}
}
idsToResponse(&req.Response, &obj)
if len(contentType) == 0 {
contentType, _, err = readContentType(obj.PayloadSize(), func(sz uint64) (io.Reader, error) {
var prmRange pool.PrmObjectRange
prmRange.SetAddress(objectAddress)
prmRange.SetLength(sz)
if btoken != nil {
prmRange.UseBearer(*btoken)
}
resObj, err := h.pool.ObjectRange(ctx, prmRange)
if err != nil {
return nil, err
}
return &resObj, nil
})
if err != nil && err != io.EOF {
req.handleFrostFSErr(err, start)
return
}
}
req.SetContentType(contentType)
}
func idsToResponse(resp *fasthttp.Response, obj *object.Object) {
objID, _ := obj.ID()
cnrID, _ := obj.ContainerID()
resp.Header.Set(hdrObjectID, objID.String())
resp.Header.Set(hdrOwnerID, obj.OwnerID().String())
resp.Header.Set(hdrContainerID, cnrID.String())
}
// HeadByAddressOrBucketName handles head requests using simple cid/oid or bucketname/key format.
func (h *Handler) HeadByAddressOrBucketName(c *fasthttp.RequestCtx) {
test, _ := c.UserValue("oid").(string)
var id oid.ID
err := id.DecodeString(test)
if err != nil {
h.byBucketname(c, h.headObject)
} else {
h.byAddress(c, h.headObject)
}
}
// HeadByAttribute handles attribute-based head requests.
func (h *Handler) HeadByAttribute(c *fasthttp.RequestCtx) {
h.byAttribute(c, h.headObject)
}

View file

@ -0,0 +1,47 @@
package handler
import (
"io"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/multipart"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
"go.uber.org/zap"
)
// MultipartFile provides standard ReadCloser interface and also allows one to
// get file name, it's used for multipart uploads.
type MultipartFile interface {
io.ReadCloser
FileName() string
}
func fetchMultipartFile(l *zap.Logger, r io.Reader, boundary string) (MultipartFile, error) {
// To have a custom buffer (3mb) the custom multipart reader is used.
// Default reader uses 4KiB chunks, which slow down upload speed up to 400%
// https://github.com/golang/go/blob/91b9915d3f6f8cd2e9e9fda63f67772803adfa03/src/mime/multipart/multipart.go#L32
reader := multipart.NewReader(r, boundary)
for {
part, err := reader.NextPart()
if err != nil {
return nil, err
}
name := part.FormName()
if name == "" {
l.Debug(logs.IgnorePartEmptyFormName)
continue
}
filename := part.FileName()
// ignore multipart/form-data values
if filename == "" {
l.Debug(logs.IgnorePartEmptyFilename, zap.String("form", name))
continue
}
return part, nil
}
}

View file

@ -0,0 +1,423 @@
// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//
/*
Package multipart implements MIME multipart parsing, as defined in RFC
2046.
The implementation is sufficient for HTTP (RFC 2388) and the multipart
bodies generated by popular browsers.
*/
package multipart
import (
"bufio"
"bytes"
"fmt"
"io"
"mime"
"mime/quotedprintable"
"net/textproto"
"strings"
)
var emptyParams = make(map[string]string)
// This constant needs to be at least 76 for this package to work correctly.
// This is because \r\n--separator_of_len_70- would fill the buffer and it
// wouldn't be safe to consume a single byte from it.
// This constant is different from the constant in stdlib. The standard value is 4096.
const peekBufferSize = 3 << 20
// A Part represents a single part in a multipart body.
type Part struct {
// The headers of the body, if any, with the keys canonicalized
// in the same fashion that the Go http.Request headers are.
// For example, "foo-bar" changes case to "Foo-Bar"
Header textproto.MIMEHeader
mr *Reader
disposition string
dispositionParams map[string]string
// r is either a reader directly reading from mr, or it's a
// wrapper around such a reader, decoding the
// Content-Transfer-Encoding
r io.Reader
n int // known data bytes waiting in mr.bufReader
total int64 // total data bytes read already
err error // error to return when n == 0
readErr error // read error observed from mr.bufReader
}
// FormName returns the name parameter if p has a Content-Disposition
// of type "form-data". Otherwise it returns the empty string.
func (p *Part) FormName() string {
// See https://tools.ietf.org/html/rfc2183 section 2 for EBNF
// of Content-Disposition value format.
if p.dispositionParams == nil {
p.parseContentDisposition()
}
if p.disposition != "form-data" {
return ""
}
return p.dispositionParams["name"]
}
// FileName returns the filename parameter of the Part's
// Content-Disposition header.
func (p *Part) FileName() string {
if p.dispositionParams == nil {
p.parseContentDisposition()
}
return p.dispositionParams["filename"]
}
func (p *Part) parseContentDisposition() {
v := p.Header.Get("Content-Disposition")
var err error
p.disposition, p.dispositionParams, err = mime.ParseMediaType(v)
if err != nil {
p.dispositionParams = emptyParams
}
}
// NewReader creates a new multipart Reader reading from r using the
// given MIME boundary.
//
// The boundary is usually obtained from the "boundary" parameter of
// the message's "Content-Type" header. Use mime.ParseMediaType to
// parse such headers.
func NewReader(r io.Reader, boundary string) *Reader {
b := []byte("\r\n--" + boundary + "--")
return &Reader{
bufReader: bufio.NewReaderSize(&stickyErrorReader{r: r}, peekBufferSize),
nl: b[:2],
nlDashBoundary: b[:len(b)-2],
dashBoundaryDash: b[2:],
dashBoundary: b[2 : len(b)-2],
}
}
// stickyErrorReader is an io.Reader which never calls Read on its
// underlying Reader once an error has been seen. (the io.Reader
// interface's contract promises nothing about the return values of
// Read calls after an error, yet this package does do multiple Reads
// after error).
type stickyErrorReader struct {
r io.Reader
err error
}
func (r *stickyErrorReader) Read(p []byte) (n int, _ error) {
if r.err != nil {
return 0, r.err
}
n, r.err = r.r.Read(p)
return n, r.err
}
func newPart(mr *Reader, rawPart bool) (*Part, error) {
bp := &Part{
Header: make(map[string][]string),
mr: mr,
}
if err := bp.populateHeaders(); err != nil {
return nil, err
}
bp.r = partReader{bp}
// rawPart is used to switch between Part.NextPart and Part.NextRawPart.
if !rawPart {
const cte = "Content-Transfer-Encoding"
if strings.EqualFold(bp.Header.Get(cte), "quoted-printable") {
bp.Header.Del(cte)
bp.r = quotedprintable.NewReader(bp.r)
}
}
return bp, nil
}
func (p *Part) populateHeaders() error {
r := textproto.NewReader(p.mr.bufReader)
header, err := r.ReadMIMEHeader()
if err == nil {
p.Header = header
}
return err
}
// Read reads the body of a part, after its headers and before the
// next part (if any) begins.
func (p *Part) Read(d []byte) (n int, err error) {
return p.r.Read(d)
}
// partReader implements io.Reader by reading raw bytes directly from the
// wrapped *Part, without doing any Transfer-Encoding decoding.
type partReader struct {
p *Part
}
func (pr partReader) Read(d []byte) (int, error) {
p := pr.p
br := p.mr.bufReader
// Read into buffer until we identify some data to return,
// or we find a reason to stop (boundary or read error).
for p.n == 0 && p.err == nil {
peek, _ := br.Peek(br.Buffered())
p.n, p.err = scanUntilBoundary(peek, p.mr.dashBoundary, p.mr.nlDashBoundary, p.total, p.readErr)
if p.n == 0 && p.err == nil {
// Force buffered I/O to read more into buffer.
_, p.readErr = br.Peek(len(peek) + 1)
if p.readErr == io.EOF {
p.readErr = io.ErrUnexpectedEOF
}
}
}
// Read out from "data to return" part of buffer.
if p.n == 0 {
return 0, p.err
}
n := len(d)
if n > p.n {
n = p.n
}
n, _ = br.Read(d[:n])
p.total += int64(n)
p.n -= n
if p.n == 0 {
return n, p.err
}
return n, nil
}
// scanUntilBoundary scans buf to identify how much of it can be safely
// returned as part of the Part body.
// dashBoundary is "--boundary".
// nlDashBoundary is "\r\n--boundary" or "\n--boundary", depending on what mode we are in.
// The comments below (and the name) assume "\n--boundary", but either is accepted.
// total is the number of bytes read out so far. If total == 0, then a leading "--boundary" is recognized.
// readErr is the read error, if any, that followed reading the bytes in buf.
// scanUntilBoundary returns the number of data bytes from buf that can be
// returned as part of the Part body and also the error to return (if any)
// once those data bytes are done.
func scanUntilBoundary(buf, dashBoundary, nlDashBoundary []byte, total int64, readErr error) (int, error) {
if total == 0 {
// At beginning of body, allow dashBoundary.
if bytes.HasPrefix(buf, dashBoundary) {
switch matchAfterPrefix(buf, dashBoundary, readErr) {
case -1:
return len(dashBoundary), nil
case 0:
return 0, nil
case +1:
return 0, io.EOF
}
}
if bytes.HasPrefix(dashBoundary, buf) {
return 0, readErr
}
}
// Search for "\n--boundary".
if i := bytes.Index(buf, nlDashBoundary); i >= 0 {
switch matchAfterPrefix(buf[i:], nlDashBoundary, readErr) {
case -1:
return i + len(nlDashBoundary), nil
case 0:
return i, nil
case +1:
return i, io.EOF
}
}
if bytes.HasPrefix(nlDashBoundary, buf) {
return 0, readErr
}
// Otherwise, anything up to the final \n is not part of the boundary
// and so must be part of the body.
// Also if the section from the final \n onward is not a prefix of the boundary,
// it too must be part of the body.
i := bytes.LastIndexByte(buf, nlDashBoundary[0])
if i >= 0 && bytes.HasPrefix(nlDashBoundary, buf[i:]) {
return i, nil
}
return len(buf), readErr
}
// matchAfterPrefix checks whether buf should be considered to match the boundary.
// The prefix is "--boundary" or "\r\n--boundary" or "\n--boundary",
// and the caller has verified already that bytes.HasPrefix(buf, prefix) is true.
//
// matchAfterPrefix returns +1 if the buffer does match the boundary,
// meaning the prefix is followed by a dash, space, tab, cr, nl, or end of input.
// It returns -1 if the buffer definitely does NOT match the boundary,
// meaning the prefix is followed by some other character.
// For example, "--foobar" does not match "--foo".
// It returns 0 more input needs to be read to make the decision,
// meaning that len(buf) == len(prefix) and readErr == nil.
func matchAfterPrefix(buf, prefix []byte, readErr error) int {
if len(buf) == len(prefix) {
if readErr != nil {
return +1
}
return 0
}
c := buf[len(prefix)]
if c == ' ' || c == '\t' || c == '\r' || c == '\n' || c == '-' {
return +1
}
return -1
}
func (p *Part) Close() error {
_, _ = io.Copy(io.Discard, p)
return nil
}
// Reader is an iterator over parts in a MIME multipart body.
// Reader's underlying parser consumes its input as needed. Seeking
// isn't supported.
type Reader struct {
bufReader *bufio.Reader
currentPart *Part
partsRead int
nl []byte // "\r\n" or "\n" (set after seeing first boundary line)
nlDashBoundary []byte // nl + "--boundary"
dashBoundaryDash []byte // "--boundary--"
dashBoundary []byte // "--boundary"
}
// NextPart returns the next part in the multipart or an error.
// When there are no more parts, the error io.EOF is returned.
//
// As a special case, if the "Content-Transfer-Encoding" header
// has a value of "quoted-printable", that header is instead
// hidden and the body is transparently decoded during Read calls.
func (r *Reader) NextPart() (*Part, error) {
return r.nextPart(false)
}
// NextRawPart returns the next part in the multipart or an error.
// When there are no more parts, the error io.EOF is returned.
//
// Unlike NextPart, it does not have special handling for
// "Content-Transfer-Encoding: quoted-printable".
func (r *Reader) NextRawPart() (*Part, error) {
return r.nextPart(true)
}
func (r *Reader) nextPart(rawPart bool) (*Part, error) {
if r.currentPart != nil {
r.currentPart.Close()
}
if string(r.dashBoundary) == "--" {
return nil, fmt.Errorf("multipart: boundary is empty")
}
expectNewPart := false
for {
line, err := r.bufReader.ReadSlice('\n')
if err == io.EOF && r.isFinalBoundary(line) {
// If the buffer ends in "--boundary--" without the
// trailing "\r\n", ReadSlice will return an error
// (since it's missing the '\n'), but this is a valid
// multipart EOF so we need to return io.EOF instead of
// a fmt-wrapped one.
return nil, io.EOF
}
if err != nil {
return nil, fmt.Errorf("multipart: NextPart: %v", err)
}
if r.isBoundaryDelimiterLine(line) {
r.partsRead++
bp, err := newPart(r, rawPart)
if err != nil {
return nil, err
}
r.currentPart = bp
return bp, nil
}
if r.isFinalBoundary(line) {
// Expected EOF
return nil, io.EOF
}
if expectNewPart {
return nil, fmt.Errorf("multipart: expecting a new Part; got line %q", string(line))
}
if r.partsRead == 0 {
// skip line
continue
}
// Consume the "\n" or "\r\n" separator between the
// body of the previous part and the boundary line we
// now expect will follow. (either a new part or the
// end boundary)
if bytes.Equal(line, r.nl) {
expectNewPart = true
continue
}
return nil, fmt.Errorf("multipart: unexpected line in Next(): %q", line)
}
}
// isFinalBoundary reports whether line is the final boundary line
// indicating that all parts are over.
// It matches `^--boundary--[ \t]*(\r\n)?$`.
func (r *Reader) isFinalBoundary(line []byte) bool {
if !bytes.HasPrefix(line, r.dashBoundaryDash) {
return false
}
rest := line[len(r.dashBoundaryDash):]
rest = skipLWSPChar(rest)
return len(rest) == 0 || bytes.Equal(rest, r.nl)
}
func (r *Reader) isBoundaryDelimiterLine(line []byte) (ret bool) {
// https://tools.ietf.org/html/rfc2046#section-5.1
// The boundary delimiter line is then defined as a line
// consisting entirely of two hyphen characters ("-",
// decimal value 45) followed by the boundary parameter
// value from the Content-Type header field, optional linear
// whitespace, and a terminating CRLF.
if !bytes.HasPrefix(line, r.dashBoundary) {
return false
}
rest := line[len(r.dashBoundary):]
rest = skipLWSPChar(rest)
// On the first part, see our lines are ending in \n instead of \r\n
// and switch into that mode if so. This is a violation of the spec,
// but occurs in practice.
if r.partsRead == 0 && len(rest) == 1 && rest[0] == '\n' {
r.nl = r.nl[1:]
r.nlDashBoundary = r.nlDashBoundary[1:]
}
return bytes.Equal(rest, r.nl)
}
// skipLWSPChar returns b with leading spaces and tabs removed.
// RFC 822 defines:
//
// LWSP-char = SPACE / HTAB
func skipLWSPChar(b []byte) []byte {
for len(b) > 0 && (b[0] == ' ' || b[0] == '\t') {
b = b[1:]
}
return b
}

View file

@ -0,0 +1,164 @@
//go:build !integration
package handler
import (
"crypto/rand"
"fmt"
"io"
"mime/multipart"
"os"
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
"github.com/stretchr/testify/require"
"go.uber.org/zap"
)
func generateRandomFile(size int64) (string, error) {
file, err := os.CreateTemp("", "data")
if err != nil {
return "", err
}
_, err = io.CopyN(file, rand.Reader, size)
if err != nil {
return "", err
}
return file.Name(), file.Close()
}
func BenchmarkAll(b *testing.B) {
fileName, err := generateRandomFile(1024 * 1024 * 256)
require.NoError(b, err)
fmt.Println(fileName)
defer os.Remove(fileName)
b.Run("bare", func(b *testing.B) {
for i := 0; i < b.N; i++ {
err := bareRead(fileName)
require.NoError(b, err)
}
})
b.Run("default", func(b *testing.B) {
for i := 0; i < b.N; i++ {
err := defaultMultipart(fileName)
require.NoError(b, err)
}
})
b.Run("custom", func(b *testing.B) {
for i := 0; i < b.N; i++ {
err := customMultipart(fileName)
require.NoError(b, err)
}
})
}
func defaultMultipart(filename string) error {
r, bound := multipartFile(filename)
logger, err := zap.NewProduction()
if err != nil {
return err
}
file, err := fetchMultipartFileDefault(logger, r, bound)
if err != nil {
return err
}
_, err = io.Copy(io.Discard, file)
return err
}
func TestName(t *testing.T) {
fileName, err := generateRandomFile(1024 * 1024 * 256)
require.NoError(t, err)
fmt.Println(fileName)
defer os.Remove(fileName)
err = defaultMultipart(fileName)
require.NoError(t, err)
}
func customMultipart(filename string) error {
r, bound := multipartFile(filename)
logger, err := zap.NewProduction()
if err != nil {
return err
}
file, err := fetchMultipartFile(logger, r, bound)
if err != nil {
return err
}
_, err = io.Copy(io.Discard, file)
return err
}
func fetchMultipartFileDefault(l *zap.Logger, r io.Reader, boundary string) (MultipartFile, error) {
reader := multipart.NewReader(r, boundary)
for {
part, err := reader.NextPart()
if err != nil {
return nil, err
}
name := part.FormName()
if name == "" {
l.Debug(logs.IgnorePartEmptyFormName)
continue
}
filename := part.FileName()
// ignore multipart/form-data values
if filename == "" {
l.Debug(logs.IgnorePartEmptyFilename, zap.String("form", name))
continue
}
return part, nil
}
}
func bareRead(filename string) error {
r, _ := multipartFile(filename)
_, err := io.Copy(io.Discard, r)
return err
}
func multipartFile(filename string) (*io.PipeReader, string) {
r, w := io.Pipe()
m := multipart.NewWriter(w)
go func() {
defer w.Close()
defer m.Close()
part, err := m.CreateFormFile("myFile", "foo.txt")
if err != nil {
fmt.Println(err)
return
}
file, err := os.Open(filename)
if err != nil {
fmt.Println(err)
return
}
defer file.Close()
if _, err = io.Copy(part, file); err != nil {
fmt.Println(err)
return
}
}()
return r, m.Boundary()
}

141
internal/handler/reader.go Normal file
View file

@ -0,0 +1,141 @@
package handler
import (
"bytes"
"context"
"io"
"net/http"
"path"
"strconv"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/response"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
"github.com/valyala/fasthttp"
"go.uber.org/zap"
)
type readCloser struct {
io.Reader
io.Closer
}
// initializes io.Reader with the limited size and detects Content-Type from it.
// Returns r's error directly. Also returns the processed data.
func readContentType(maxSize uint64, rInit func(uint64) (io.Reader, error)) (string, []byte, error) {
if maxSize > sizeToDetectType {
maxSize = sizeToDetectType
}
buf := make([]byte, maxSize) // maybe sync-pool the slice?
r, err := rInit(maxSize)
if err != nil {
return "", nil, err
}
n, err := r.Read(buf)
if err != nil && err != io.EOF {
return "", nil, err
}
buf = buf[:n]
return http.DetectContentType(buf), buf, err // to not lose io.EOF
}
func (h *Handler) receiveFile(ctx context.Context, req request, objectAddress oid.Address) {
var (
err error
dis = "inline"
start = time.Now()
filename string
)
var prm pool.PrmObjectGet
prm.SetAddress(objectAddress)
if btoken := bearerToken(ctx); btoken != nil {
prm.UseBearer(*btoken)
}
rObj, err := h.pool.GetObject(ctx, prm)
if err != nil {
req.handleFrostFSErr(err, start)
return
}
// we can't close reader in this function, so how to do it?
if req.Request.URI().QueryArgs().GetBool("download") {
dis = "attachment"
}
payloadSize := rObj.Header.PayloadSize()
req.Response.Header.Set(fasthttp.HeaderContentLength, strconv.FormatUint(payloadSize, 10))
var contentType string
for _, attr := range rObj.Header.Attributes() {
key := attr.Key()
val := attr.Value()
if !isValidToken(key) || !isValidValue(val) {
continue
}
key = utils.BackwardTransformIfSystem(key)
req.Response.Header.Set(utils.UserAttributeHeaderPrefix+key, val)
switch key {
case object.AttributeFileName:
filename = val
case object.AttributeTimestamp:
value, err := strconv.ParseInt(val, 10, 64)
if err != nil {
req.log.Info(logs.CouldntParseCreationDate,
zap.String("key", key),
zap.String("val", val),
zap.Error(err))
continue
}
req.Response.Header.Set(fasthttp.HeaderLastModified,
time.Unix(value, 0).UTC().Format(http.TimeFormat))
case object.AttributeContentType:
contentType = val
}
}
idsToResponse(&req.Response, &rObj.Header)
if len(contentType) == 0 {
// determine the Content-Type from the payload head
var payloadHead []byte
contentType, payloadHead, err = readContentType(payloadSize, func(uint64) (io.Reader, error) {
return rObj.Payload, nil
})
if err != nil && err != io.EOF {
req.log.Error(logs.CouldNotDetectContentTypeFromPayload, zap.Error(err))
response.Error(req.RequestCtx, "could not detect Content-Type from payload: "+err.Error(), fasthttp.StatusBadRequest)
return
}
// reset payload reader since a part of the data has been read
var headReader io.Reader = bytes.NewReader(payloadHead)
if err != io.EOF { // otherwise, we've already read full payload
headReader = io.MultiReader(headReader, rObj.Payload)
}
// note: we could do with io.Reader, but SetBodyStream below closes body stream
// if it implements io.Closer and that's useful for us.
rObj.Payload = readCloser{headReader, rObj.Payload}
}
req.SetContentType(contentType)
req.Response.Header.Set(fasthttp.HeaderContentDisposition, dis+"; filename="+path.Base(filename))
req.Response.SetBodyStream(rObj.Payload, int(payloadSize))
}

View file

@ -0,0 +1,48 @@
//go:build !integration
package handler
import (
"io"
"strings"
"testing"
"github.com/stretchr/testify/require"
)
func TestDetector(t *testing.T) {
txtContentType := "text/plain; charset=utf-8"
sb := strings.Builder{}
for i := 0; i < 10; i++ {
sb.WriteString("Some txt content. Content-Type must be detected properly by detector.")
}
for _, tc := range []struct {
Name string
ContentType string
Expected string
}{
{
Name: "less than 512b",
ContentType: txtContentType,
Expected: sb.String()[:256],
},
{
Name: "more than 512b",
ContentType: txtContentType,
Expected: sb.String(),
},
} {
t.Run(tc.Name, func(t *testing.T) {
contentType, data, err := readContentType(uint64(len(tc.Expected)),
func(sz uint64) (io.Reader, error) {
return strings.NewReader(tc.Expected), nil
},
)
require.NoError(t, err)
require.Equal(t, tc.ContentType, contentType)
require.True(t, strings.HasPrefix(tc.Expected, string(data)))
})
}
}

190
internal/handler/upload.go Normal file
View file

@ -0,0 +1,190 @@
package handler
import (
"context"
"encoding/json"
"io"
"net/http"
"strconv"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/response"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
"github.com/valyala/fasthttp"
"go.uber.org/zap"
)
const (
jsonHeader = "application/json; charset=UTF-8"
drainBufSize = 4096
)
type putResponse struct {
ObjectID string `json:"object_id"`
ContainerID string `json:"container_id"`
}
func newPutResponse(addr oid.Address) *putResponse {
return &putResponse{
ObjectID: addr.Object().EncodeToString(),
ContainerID: addr.Container().EncodeToString(),
}
}
func (pr *putResponse) encode(w io.Writer) error {
enc := json.NewEncoder(w)
enc.SetIndent("", "\t")
return enc.Encode(pr)
}
// Upload handles multipart upload request.
func (h *Handler) Upload(req *fasthttp.RequestCtx) {
var (
file MultipartFile
idObj oid.ID
addr oid.Address
scid, _ = req.UserValue("cid").(string)
log = h.log.With(zap.String("cid", scid))
bodyStream = req.RequestBodyStream()
drainBuf = make([]byte, drainBufSize)
)
ctx := utils.GetContextFromRequest(req)
idCnr, err := h.getContainerID(ctx, scid)
if err != nil {
log.Error(logs.WrongContainerID, zap.Error(err))
response.Error(req, "wrong container id", fasthttp.StatusBadRequest)
return
}
defer func() {
// If the temporary reader can be closed - let's close it.
if file == nil {
return
}
err := file.Close()
log.Debug(
logs.CloseTemporaryMultipartFormFile,
zap.Stringer("address", addr),
zap.String("filename", file.FileName()),
zap.Error(err),
)
}()
boundary := string(req.Request.Header.MultipartFormBoundary())
if file, err = fetchMultipartFile(h.log, bodyStream, boundary); err != nil {
log.Error(logs.CouldNotReceiveMultipartForm, zap.Error(err))
response.Error(req, "could not receive multipart/form: "+err.Error(), fasthttp.StatusBadRequest)
return
}
filtered, err := filterHeaders(h.log, &req.Request.Header)
if err != nil {
log.Error(logs.CouldNotProcessHeaders, zap.Error(err))
response.Error(req, err.Error(), fasthttp.StatusBadRequest)
return
}
now := time.Now()
if rawHeader := req.Request.Header.Peek(fasthttp.HeaderDate); rawHeader != nil {
if parsed, err := time.Parse(http.TimeFormat, string(rawHeader)); err != nil {
log.Warn(logs.CouldNotParseClientTime, zap.String("Date header", string(rawHeader)), zap.Error(err))
} else {
now = parsed
}
}
if err = utils.PrepareExpirationHeader(req, h.pool, filtered, now); err != nil {
log.Error(logs.CouldNotPrepareExpirationHeader, zap.Error(err))
response.Error(req, "could not prepare expiration header: "+err.Error(), fasthttp.StatusBadRequest)
return
}
attributes := make([]object.Attribute, 0, len(filtered))
// prepares attributes from filtered headers
for key, val := range filtered {
attribute := object.NewAttribute()
attribute.SetKey(key)
attribute.SetValue(val)
attributes = append(attributes, *attribute)
}
// sets FileName attribute if it wasn't set from header
if _, ok := filtered[object.AttributeFileName]; !ok {
filename := object.NewAttribute()
filename.SetKey(object.AttributeFileName)
filename.SetValue(file.FileName())
attributes = append(attributes, *filename)
}
// sets Timestamp attribute if it wasn't set from header and enabled by settings
if _, ok := filtered[object.AttributeTimestamp]; !ok && h.settings.DefaultTimestamp() {
timestamp := object.NewAttribute()
timestamp.SetKey(object.AttributeTimestamp)
timestamp.SetValue(strconv.FormatInt(time.Now().Unix(), 10))
attributes = append(attributes, *timestamp)
}
obj := object.New()
obj.SetContainerID(*idCnr)
obj.SetOwnerID(h.ownerID)
obj.SetAttributes(attributes...)
var prm pool.PrmObjectPut
prm.SetHeader(*obj)
prm.SetPayload(file)
bt := h.fetchBearerToken(ctx)
if bt != nil {
prm.UseBearer(*bt)
}
if idObj, err = h.pool.PutObject(ctx, prm); err != nil {
h.handlePutFrostFSErr(req, err)
return
}
addr.SetObject(idObj)
addr.SetContainer(*idCnr)
// Try to return the response, otherwise, if something went wrong, throw an error.
if err = newPutResponse(addr).encode(req); err != nil {
log.Error(logs.CouldNotEncodeResponse, zap.Error(err))
response.Error(req, "could not encode response", fasthttp.StatusBadRequest)
return
}
// Multipart is multipart and thus can contain more than one part which
// we ignore at the moment. Also, when dealing with chunked encoding
// the last zero-length chunk might be left unread (because multipart
// reader only cares about its boundary and doesn't look further) and
// it will be (erroneously) interpreted as the start of the next
// pipelined header. Thus we need to drain the body buffer.
for {
_, err = bodyStream.Read(drainBuf)
if err == io.EOF || err == io.ErrUnexpectedEOF {
break
}
}
// Report status code and content type.
req.Response.SetStatusCode(fasthttp.StatusOK)
req.Response.Header.SetContentType(jsonHeader)
}
func (h *Handler) handlePutFrostFSErr(r *fasthttp.RequestCtx, err error) {
statusCode, msg, additionalFields := response.FormErrorResponse("could not store file in frostfs", err)
logFields := append([]zap.Field{zap.Error(err)}, additionalFields...)
h.log.Error(logs.CouldNotStoreFileInFrostfs, logFields...)
response.Error(r, msg, statusCode)
}
func (h *Handler) fetchBearerToken(ctx context.Context) *bearer.Token {
if tkn, err := tokens.LoadBearerToken(ctx); err == nil && tkn != nil {
return tkn
}
return nil
}

60
internal/handler/utils.go Normal file
View file

@ -0,0 +1,60 @@
package handler
import (
"context"
"strings"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/response"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
"github.com/valyala/fasthttp"
"go.uber.org/zap"
)
type request struct {
*fasthttp.RequestCtx
log *zap.Logger
}
func (r *request) handleFrostFSErr(err error, start time.Time) {
logFields := []zap.Field{
zap.Stringer("elapsed", time.Since(start)),
zap.Error(err),
}
statusCode, msg, additionalFields := response.FormErrorResponse("could not receive object", err)
logFields = append(logFields, additionalFields...)
r.log.Error(logs.CouldNotReceiveObject, logFields...)
response.Error(r.RequestCtx, msg, statusCode)
}
func bearerToken(ctx context.Context) *bearer.Token {
if tkn, err := tokens.LoadBearerToken(ctx); err == nil {
return tkn
}
return nil
}
func isValidToken(s string) bool {
for _, c := range s {
if c <= ' ' || c > 127 {
return false
}
if strings.ContainsRune("()<>@,;:\\\"/[]?={}", c) {
return false
}
}
return true
}
func isValidValue(s string) bool {
for _, c := range s {
// HTTP specification allows for more technically, but we don't want to escape things.
if c < ' ' || c > 127 || c == '"' {
return false
}
}
return true
}