From 3f635a018aa043ba465a52b9c2a9a009b1285914 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Sat, 13 Feb 2021 19:17:01 +0300 Subject: [PATCH] Refactoring uploading - DisablePreParseMultipartForm = true - used `fetchMultipartFile` method instead of `MultipartForm` - don't store temporary files, only streaming Signed-off-by: Evgeniy Kulikov --- app.go | 2 +- multipart.go | 41 ++++++++++++++++++++++++++++++++++++++++ upload.go | 53 ++++++++++------------------------------------------ 3 files changed, 52 insertions(+), 44 deletions(-) create mode 100644 multipart.go diff --git a/app.go b/app.go index 8e4a961..6c70b90 100644 --- a/app.go +++ b/app.go @@ -102,7 +102,7 @@ func newApp(ctx context.Context, opt ...Option) App { // FIXME don't work with StreamRequestBody, // some bugs with readMultipartForm // https://github.com/valyala/fasthttp/issues/968 - // a.web.DisablePreParseMultipartForm = true + a.web.DisablePreParseMultipartForm = true a.web.StreamRequestBody = a.cfg.GetBool(cfgWebStreamRequestBody) // -- -- -- -- -- -- -- -- -- -- diff --git a/multipart.go b/multipart.go new file mode 100644 index 0000000..f12442f --- /dev/null +++ b/multipart.go @@ -0,0 +1,41 @@ +package main + +import ( + "io" + "mime/multipart" + + "go.uber.org/zap" +) + +type MultipartFile interface { + io.ReadCloser + FileName() string +} + +func fetchMultipartFile(l *zap.Logger, r io.Reader, boundary string) (MultipartFile, error) { + reader := multipart.NewReader(r, boundary) + + for { + part, err := reader.NextPart() + if err != nil { + return nil, err + } + + name := part.FormName() + if name == "" { + l.Debug("ignore part, empty form name") + continue + } + + filename := part.FileName() + + // ignore multipart/form-data values + if filename == "" { + l.Debug("ignore part, empty filename", zap.String("form", name)) + + continue + } + + return part, nil + } +} diff --git a/upload.go b/upload.go index 38d53e9..0bf99b2 100644 --- a/upload.go +++ b/upload.go @@ -3,7 +3,6 @@ package main import ( "encoding/json" "io" - "mime/multipart" "strconv" "time" @@ -37,10 +36,8 @@ func (pr *putResponse) Encode(w io.Writer) error { func (a *app) upload(c *fasthttp.RequestCtx) { var ( err error - name string - tmp io.Reader + file MultipartFile addr *object.Address - form *multipart.Form cid = container.NewID() sCID, _ = c.UserValue("cid").(string) @@ -55,54 +52,24 @@ func (a *app) upload(c *fasthttp.RequestCtx) { defer func() { // if temporary reader can be closed - close it - if closer, ok := tmp.(io.Closer); ok && closer != nil { - log.Debug("close temporary multipart/form file", zap.Error(closer.Close())) - } - - if form == nil { + if file == nil { return } - log.Debug("cleanup multipart/form", zap.Error(form.RemoveAll())) + log.Debug("close temporary multipart/form file", + zap.Stringer("address", addr), + zap.String("filename", file.FileName()), + zap.Error(file.Close())) }() - // tries to receive multipart/form or throw error - if form, err = c.MultipartForm(); err != nil { + boundary := string(c.Request.Header.MultipartFormBoundary()) + if file, err = fetchMultipartFile(a.log, c.RequestBodyStream(), boundary); err != nil { log.Error("could not receive multipart/form", zap.Error(err)) c.Error("could not receive multipart/form: "+err.Error(), fasthttp.StatusBadRequest) return } - // checks that received multipart/form contains only one `file` per request - if ln := len(form.File); ln != 1 { - log.Error("received multipart/form with more then one file", zap.Int("count", ln)) - c.Error("received multipart/form with more then one file", fasthttp.StatusBadRequest) - - return - } - - for _, file := range form.File { - // because multipart/form can contains multiple FileHeader records - // we should check that we have only one per request or throw error - if ln := len(file); ln != 1 { - log.Error("received multipart/form file should contains one record", zap.Int("count", ln)) - c.Error("received multipart/form file should contains one record", fasthttp.StatusBadRequest) - - return - } - - name = file[0].Filename - - // opens multipart/form file to work within or throw error - if tmp, err = file[0].Open(); err != nil { - log.Error("could not prepare uploaded file", zap.Error(err)) - c.Error("could not prepare uploaded file", fasthttp.StatusBadRequest) - - return - } - } - filtered := filterHeaders(a.log, &c.Request.Header) attributes := make([]*object.Attribute, 0, len(filtered)) @@ -119,7 +86,7 @@ func (a *app) upload(c *fasthttp.RequestCtx) { if _, ok := filtered[object.AttributeFileName]; !ok { filename := object.NewAttribute() filename.SetKey(object.AttributeFileName) - filename.SetValue(name) + filename.SetValue(file.FileName()) attributes = append(attributes, filename) } @@ -140,7 +107,7 @@ func (a *app) upload(c *fasthttp.RequestCtx) { raw.SetAttributes(attributes...) // tries to put file into NeoFS or throw error - if addr, err = a.cli.Object().Put(c, raw.Object(), sdk.WithPutReader(tmp)); err != nil { + if addr, err = a.cli.Object().Put(c, raw.Object(), sdk.WithPutReader(file)); err != nil { log.Error("could not store file in NeoFS", zap.Error(err)) c.Error("could not store file in NeoFS", fasthttp.StatusBadRequest)