Split DownloadZipped handler on methods. Add handler DownloadTar for downloading tar.gz archives. Make methods more universal for using in both implementations

Signed-off-by: Nikita Zinkevich <n.zinkevich@yadro.com>

@@ -130,2 +130,2 @@
-	// Zip compression.
-	cfgZipCompression = "zip.compression"
+	// Archive compression.
+	cfgArchiveCompression = "archive.compression"

@@ -65,0 +61,4 @@
+| Common headers                | See [bearer token](#bearer-token).                                                                                                                                        |
+| `X-Attribute-System-*`        | Used to set system FrostFS object attributes <br/> (e.g. use "X-Attribute-System-Expiration-Epoch" to set `__SYSTEM__EXPIRATION_EPOCH` attribute).                        |
+| `X-Attribute-*`               | Used to set regular object attributes <br/> (e.g. use "X-Attribute-My-Tag" to set `My-Tag` attribute).                                                                    |
+| `X-Attribute-Explode-Archive` | If this header is set, gate reads files from uploaded `tar.gz` archive and creates object for each file in it. Set FilePath attribute as relative path from archive root. |

@@ -115,3 +140,1 @@
-	c.Response.Header.Set(fasthttp.HeaderContentType, "application/zip")
-	c.Response.Header.Set(fasthttp.HeaderContentDisposition, "attachment; filename=\"archive.zip\"")
-	c.Response.SetStatusCode(http.StatusOK)
+	c.Response.Header.Set(fasthttp.HeaderContentType, "application/x-gzip")

@@ -165,1 +197,3 @@
-	}
+func (h *Handler) putObjectToArchive(ctx context.Context, log *zap.Logger, cnrID cid.ID, bufZip *[]byte, createArchiveHeader func(obj *object.Object) (io.Writer, error)) func(id oid.ID) bool {
+	return func(id oid.ID) bool {
+		log := log.With(zap.String("oid", id.EncodeToString()))

@@ -86,2 +88,4 @@
 	}
 
+	if header := c.Request.Header.Peek(utils.UserAttributeHeaderPrefix + explodeArchiveHeader); header != nil {
+		h.explodeGzip(c, log, bktInfo, file)

@@ -656,6 +656,7 @@ func (a *app) configureRouter(handler *handler.Handler) {
 	r.OPTIONS("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.addPreflight())
 	a.log.Info(logs.AddedPathGetByAttributeCidAttrKeyAttrVal)
 	r.GET("/zip/{cid}/{prefix:*}", a.addMiddlewares(handler.DownloadZipped))
+	r.GET("/tar/{cid}/{prefix:*}", a.addMiddlewares(handler.DownloadTar))

@@ -68,0 +95,4 @@
+		if errIter != nil {
+			log.Error(logs.IteratingOverSelectedObjectsFailed, zap.Error(errIter))
+		} else if bufZip == nil {
+			log.Error(logs.ObjectsNotFound)

@@ -68,0 +97,4 @@
+		} else if bufZip == nil {
+			log.Error(logs.ObjectsNotFound)
+		}
+		if err := zipWriter.Close(); err != nil {

@@ -123,0 +151,4 @@
+		if h.config.ArchiveCompression() {
+			gzipWriter, _ = gzip.NewWriterLevel(w, gzip.DefaultCompression)
+		} else {
+			gzipWriter, _ = gzip.NewWriterLevel(w, gzip.NoCompression)

@@ -163,3 +197,1 @@
-		},
-		Address: addr,
-	}
+func (h *Handler) putObjectToArchive(ctx context.Context, log *zap.Logger, cnrID cid.ID, bufZip *[]byte, createArchiveHeader func(obj *object.Object) (io.Writer, error)) func(id oid.ID) bool {

@@ -176,1 +254,4 @@
+	return resSearch, nil
+}
 
+func (h *Handler) writeToArchive(resGet *Object, objWriter io.Writer, bufZip []byte) error {

@@ -152,0 +224,4 @@
+	// default reader - without gzip decompression
+	accReader := newAccumulatedReader(formFile)
+	var reader io.Reader
+	if gzipReader, err := gzip.NewReader(accReader); err == nil {

@@ -0,0 +1 @@
+package handler

@@ -179,0 +239,4 @@
+			return
+		}
+		// add filepath attribute
+		attributes = append(attributes, *newAttribute(utils.UserAttributeHeaderPrefix+object.AttributeFilePath, obj.Name))

@@ -95,0 +133,4 @@
+			return
+		} else if objectsWritten == 0 {
+			log.Warn(logs.ObjectsNotFound)
+			return

@@ -213,3 +300,1 @@
-		return fmt.Errorf("flush zip writer: %v", err)
-	}
-
+	// TODO: проверить zip.Flush

@@ -15,2 +16,4 @@
 	ObjectsNotFound                                                       = "objects not found"                                                                      // Error in ../../downloader/download.go
 	CloseZipWriter                                                        = "close zip writer"                                                                       // Error in ../../downloader/download.go
+	CloseGzipWriter                                                       = "close gzip writer"                                                                      // Error in ../../downloader/download.go
+	CloseTarWriter                                                        = "close tar writer"                                                                       // Error in ../../downloader/download.go

@@ -65,0 +61,4 @@
+| Common headers         | See [bearer token](#bearer-token).                                                                                                                                                                              |
+| `X-Attribute-System-*` | Used to set system FrostFS object attributes <br/> (e.g. use "X-Attribute-System-Expiration-Epoch" to set `__SYSTEM__EXPIRATION_EPOCH` attribute).                                                              |
+| `X-Attribute-*`        | Used to set regular object attributes <br/> (e.g. use "X-Attribute-My-Tag" to set `My-Tag` attribute).                                                                                                          |
+| `X-Explode-Archive`    | If set, gate tries to read files from uploading `tar` archive and creates object for each file in it. Uploading `tar` could be compressed via Gzip. Sets FilePath attribute as relative path from archive root. |

@@ -179,0 +232,4 @@
+		}
+
+		fileName := filepath.Base(obj.Name)
+		attributes, err := h.extractAttributes(c, log, fileName)

@@ -93,1 +100,4 @@
 }
+
+// setIfNotExist sets key value to map if key is not present yet.
+func setIfNotExist(m map[string]string, key, value string) {

@@ -179,0 +187,4 @@
+	return attributes, nil
+}
+
+func newAttribute(key string, val string) *object.Attribute {

@@ -179,0 +217,4 @@
+	attrlen := len(attributes)
+	filePathIndex, fileNameIndex := attrlen-1, attrlen-2
+
+	var reader = file

@@ -179,0 +212,4 @@
+	}
+	attributes = append(attributes,
+		*newAttribute(object.AttributeFileName, ""),
+		*newAttribute(object.AttributeFilePath, ""),

@@ -150,3 +191,2 @@
 
-		var bufZip []byte
-		var addr oid.Address
+		gzipWriter, _ := gzip.NewWriterLevel(w, compressionLevel)

@@ -179,3 +179,3 @@
 func (s *appSettings) update(v *viper.Viper, l *zap.Logger) {
 	defaultTimestamp := v.GetBool(cfgUploaderHeaderEnableDefaultTimestamp)
-	zipCompression := v.GetBool(cfgZipCompression)
+	archiveCompression := v.GetBool(cfgArchiveCompression)

@@ -189,3 +234,1 @@
-		},
-		Address: addr,
-	}
+func (h *Handler) putObjectToArchive(ctx context.Context, log *zap.Logger, cnrID cid.ID, bufZip []byte, createArchiveHeader func(obj *object.Object) (io.Writer, error)) func(id oid.ID) bool {

@@ -266,3 +266,3 @@
 	addr.SetObject(objID)
 
-	handler(ctx, h.newRequest(c, log), addr)
+	handler(ctx, newRequest(c, log), addr)