[#73] Uploader, downloader structures refactoring

Signed-off-by: Marina Biryukova <m.biryukova@yadro.com>

internal/handler/download.go

@@ -0,0 +1,210 @@
+package handler
+
+import (
+	"archive/zip"
+	"bufio"
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"time"
+
+	"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
+	"git.frostfs.info/TrueCloudLab/frostfs-http-gw/response"
+	"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
+	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
+	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
+	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
+	cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
+	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
+	oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
+	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
+	"github.com/valyala/fasthttp"
+	"go.uber.org/zap"
+)
+
+// DownloadByAddressOrBucketName handles download requests using simple cid/oid or bucketname/key format.
+func (h *Handler) DownloadByAddressOrBucketName(c *fasthttp.RequestCtx) {
+	test, _ := c.UserValue("oid").(string)
+	var id oid.ID
+	err := id.DecodeString(test)
+	if err != nil {
+		h.byBucketname(c, h.receiveFile)
+	} else {
+		h.byAddress(c, h.receiveFile)
+	}
+}
+
+func (h *Handler) newRequest(ctx *fasthttp.RequestCtx, log *zap.Logger) *request {
+	return &request{
+		RequestCtx: ctx,
+		log:        log,
+	}
+}
+
+// DownloadByAttribute handles attribute-based download requests.
+func (h *Handler) DownloadByAttribute(c *fasthttp.RequestCtx) {
+	h.byAttribute(c, h.receiveFile)
+}
+
+func (h *Handler) search(ctx context.Context, cid *cid.ID, key, val string, op object.SearchMatchType) (pool.ResObjectSearch, error) {
+	filters := object.NewSearchFilters()
+	filters.AddRootFilter()
+	filters.AddFilter(key, val, op)
+
+	var prm pool.PrmObjectSearch
+	prm.SetContainerID(*cid)
+	prm.SetFilters(filters)
+	if btoken := bearerToken(ctx); btoken != nil {
+		prm.UseBearer(*btoken)
+	}
+
+	return h.pool.SearchObjects(ctx, prm)
+}
+
+func (h *Handler) getContainer(ctx context.Context, cnrID cid.ID) (container.Container, error) {
+	var prm pool.PrmContainerGet
+	prm.SetContainerID(cnrID)
+
+	return h.pool.GetContainer(ctx, prm)
+}
+
+func (h *Handler) addObjectToZip(zw *zip.Writer, obj *object.Object) (io.Writer, error) {
+	method := zip.Store
+	if h.settings.ZipCompression() {
+		method = zip.Deflate
+	}
+
+	filePath := getZipFilePath(obj)
+	if len(filePath) == 0 || filePath[len(filePath)-1] == '/' {
+		return nil, fmt.Errorf("invalid filepath '%s'", filePath)
+	}
+
+	return zw.CreateHeader(&zip.FileHeader{
+		Name:     filePath,
+		Method:   method,
+		Modified: time.Now(),
+	})
+}
+
+// DownloadZipped handles zip by prefix requests.
+func (h *Handler) DownloadZipped(c *fasthttp.RequestCtx) {
+	scid, _ := c.UserValue("cid").(string)
+	prefix, _ := url.QueryUnescape(c.UserValue("prefix").(string))
+	log := h.log.With(zap.String("cid", scid), zap.String("prefix", prefix))
+
+	ctx := utils.GetContextFromRequest(c)
+
+	containerID, err := h.getContainerID(ctx, scid)
+	if err != nil {
+		log.Error(logs.WrongContainerID, zap.Error(err))
+		response.Error(c, "wrong container id", fasthttp.StatusBadRequest)
+		return
+	}
+
+	// check if container exists here to be able to return 404 error,
+	// otherwise we get this error only in object iteration step
+	// and client get 200 OK.
+	if _, err = h.getContainer(ctx, *containerID); err != nil {
+		log.Error(logs.CouldNotCheckContainerExistence, zap.Error(err))
+		if client.IsErrContainerNotFound(err) {
+			response.Error(c, "Not Found", fasthttp.StatusNotFound)
+			return
+		}
+		response.Error(c, "could not check container existence: "+err.Error(), fasthttp.StatusBadRequest)
+		return
+	}
+
+	resSearch, err := h.search(ctx, containerID, object.AttributeFilePath, prefix, object.MatchCommonPrefix)
+	if err != nil {
+		log.Error(logs.CouldNotSearchForObjects, zap.Error(err))
+		response.Error(c, "could not search for objects: "+err.Error(), fasthttp.StatusBadRequest)
+		return
+	}
+
+	c.Response.Header.Set(fasthttp.HeaderContentType, "application/zip")
+	c.Response.Header.Set(fasthttp.HeaderContentDisposition, "attachment; filename=\"archive.zip\"")
+	c.Response.SetStatusCode(http.StatusOK)
+
+	c.SetBodyStreamWriter(func(w *bufio.Writer) {
+		defer resSearch.Close()
+
+		zipWriter := zip.NewWriter(w)
+
+		var bufZip []byte
+		var addr oid.Address
+
+		empty := true
+		called := false
+		btoken := bearerToken(ctx)
+		addr.SetContainer(*containerID)
+
+		errIter := resSearch.Iterate(func(id oid.ID) bool {
+			called = true
+
+			if empty {
+				bufZip = make([]byte, 3<<20) // the same as for upload
+			}
+			empty = false
+
+			addr.SetObject(id)
+			if err = h.zipObject(ctx, zipWriter, addr, btoken, bufZip); err != nil {
+				log.Error(logs.FailedToAddObjectToArchive, zap.String("oid", id.EncodeToString()), zap.Error(err))
+			}
+
+			return false
+		})
+		if errIter != nil {
+			log.Error(logs.IteratingOverSelectedObjectsFailed, zap.Error(errIter))
+		} else if !called {
+			log.Error(logs.ObjectsNotFound)
+		}
+
+		if err = zipWriter.Close(); err != nil {
+			log.Error(logs.CloseZipWriter, zap.Error(err))
+		}
+	})
+}
+
+func (h *Handler) zipObject(ctx context.Context, zipWriter *zip.Writer, addr oid.Address, btoken *bearer.Token, bufZip []byte) error {
+	var prm pool.PrmObjectGet
+	prm.SetAddress(addr)
+	if btoken != nil {
+		prm.UseBearer(*btoken)
+	}
+
+	resGet, err := h.pool.GetObject(ctx, prm)
+	if err != nil {
+		return fmt.Errorf("get FrostFS object: %v", err)
+	}
+
+	objWriter, err := h.addObjectToZip(zipWriter, &resGet.Header)
+	if err != nil {
+		return fmt.Errorf("zip create header: %v", err)
+	}
+
+	if _, err = io.CopyBuffer(objWriter, resGet.Payload, bufZip); err != nil {
+		return fmt.Errorf("copy object payload to zip file: %v", err)
+	}
+
+	if err = resGet.Payload.Close(); err != nil {
+		return fmt.Errorf("object body close error: %w", err)
+	}
+
+	if err = zipWriter.Flush(); err != nil {
+		return fmt.Errorf("flush zip writer: %v", err)
+	}
+
+	return nil
+}
+
+func getZipFilePath(obj *object.Object) string {
+	for _, attr := range obj.Attributes() {
+		if attr.Key() == object.AttributeFilePath {
+			return attr.Value()
+		}
+	}
+
+	return ""
+}