diff --git a/README.md b/README.md index d65992a..37a8866 100644 --- a/README.md +++ b/README.md @@ -156,6 +156,10 @@ if needed. All timing options accept values with suffixes, so "15s" is 15 seconds and "2m" is 2 minutes. +### Zip streaming +The gateway supports downloading files by common prefix (like dir) in zip format. You can enable compression +using config or `HTTP_GW_ZIP_COMPRESSION=true` environment variable. + ### Logging `--verbose` flag enables gRPC logging and there is a number of environment @@ -192,6 +196,9 @@ peers: 0: address: grpc://s01.neofs.devenv:8080 weight: 1 + +zip: + compression: false ``` To know nesting level of variable you need to cut off the prefix `HTTP_GW` from variable and split the rest parts by `_`. @@ -311,6 +318,12 @@ $ wget http://localhost:8082/get/Dxhf4PNprrJHWWTG5RGLdfLkJiSQ3AQqit1MSnEPRkDZ/2m ``` +##### Zip +You can download some dir (files with the same prefix) in zip (it will be compressed if config contains appropriate param): +``` +$ wget http://localhost:8082/zip/Dxhf4PNprrJHWWTG5RGLdfLkJiSQ3AQqit1MSnEPRkDZ/common/prefix +``` + #### Replies You get object contents in the reply body (if GET method was used), but at the same time you also get a diff --git a/app.go b/app.go index 91c108a..59e0924 100644 --- a/app.go +++ b/app.go @@ -190,7 +190,7 @@ func (a *app) Serve(ctx context.Context) { }() edts := a.cfg.GetBool(cfgUploaderHeaderEnableDefaultTimestamp) uploader := uploader.New(a.log, a.pool, edts) - downloader, err := downloader.New(ctx, a.log, a.pool) + downloader, err := downloader.New(a.log, downloader.Settings{ZipCompression: a.cfg.GetBool(cfgZipCompression)}, a.pool) if err != nil { a.log.Fatal("failed to create downloader", zap.Error(err)) } diff --git a/downloader/download.go b/downloader/download.go index 543e2f3..1d60805 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -229,14 +229,19 @@ func (o objectIDs) Slice() []string { // Downloader is a download request handler. type Downloader struct { - log *zap.Logger - pool pool.Pool + log *zap.Logger + pool pool.Pool + settings Settings +} + +type Settings struct { + ZipCompression bool } // New creates an instance of Downloader using specified options. -func New(ctx context.Context, log *zap.Logger, conns pool.Pool) (*Downloader, error) { +func New(log *zap.Logger, settings Settings, conns pool.Pool) (*Downloader, error) { var err error - d := &Downloader{log: log, pool: conns} + d := &Downloader{log: log, pool: conns, settings: settings} if err != nil { return nil, fmt.Errorf("failed to get neofs client's reusable artifacts: %w", err) } @@ -396,6 +401,11 @@ func (d *Downloader) DownloadZipped(c *fasthttp.RequestCtx) { func (d *Downloader) streamFiles(c *fasthttp.RequestCtx, cid *cid.ID, ids []*object.ID) error { zipWriter := zip.NewWriter(c) + compression := zip.Store + if d.settings.ZipCompression { + compression = zip.Deflate + } + for _, id := range ids { var r io.Reader readerInitCtx, initReader := context.WithCancel(c) @@ -413,7 +423,7 @@ func (d *Downloader) streamFiles(c *fasthttp.RequestCtx, cid *cid.ID, ids []*obj header := &zip.FileHeader{ Name: getFilename(obj), - Method: zip.Store, + Method: compression, Modified: time.Now(), } entryWriter, err := zipWriter.CreateHeader(header) diff --git a/settings.go b/settings.go index ceffec6..d09b43f 100644 --- a/settings.go +++ b/settings.go @@ -53,6 +53,9 @@ const ( // Peers. cfgPeers = "peers" + // Zip compression. + cfgZipCompression = "zip.compression" + // Application. cfgApplicationName = "app.name" cfgApplicationVersion = "app.version" @@ -134,6 +137,9 @@ func settings() *viper.Viper { // upload header v.SetDefault(cfgUploaderHeaderEnableDefaultTimestamp, false) + // zip: + v.SetDefault(cfgZipCompression, false) + if err := v.BindPFlags(flags); err != nil { panic(err) }