[#96] Add zip compression

Signed-off-by: Denis Kirillov <denis@nspcc.ru>
This commit is contained in:
Denis Kirillov 2021-10-29 16:11:34 +03:00 committed by Alex Vanin
parent 62dd8b1927
commit 9b92c0146c
4 changed files with 35 additions and 6 deletions

View file

@ -156,6 +156,10 @@ if needed.
All timing options accept values with suffixes, so "15s" is 15 seconds and
"2m" is 2 minutes.
### Zip streaming
The gateway supports downloading files by common prefix (like dir) in zip format. You can enable compression
using config or `HTTP_GW_ZIP_COMPRESSION=true` environment variable.
### Logging
`--verbose` flag enables gRPC logging and there is a number of environment
@ -192,6 +196,9 @@ peers:
0:
address: grpc://s01.neofs.devenv:8080
weight: 1
zip:
compression: false
```
To know nesting level of variable you need to cut off the prefix `HTTP_GW` from variable and split the rest parts by `_`.
@ -311,6 +318,12 @@ $ wget http://localhost:8082/get/Dxhf4PNprrJHWWTG5RGLdfLkJiSQ3AQqit1MSnEPRkDZ/2m
```
##### Zip
You can download some dir (files with the same prefix) in zip (it will be compressed if config contains appropriate param):
```
$ wget http://localhost:8082/zip/Dxhf4PNprrJHWWTG5RGLdfLkJiSQ3AQqit1MSnEPRkDZ/common/prefix
```
#### Replies
You get object contents in the reply body (if GET method was used), but at the same time you also get a

2
app.go
View file

@ -190,7 +190,7 @@ func (a *app) Serve(ctx context.Context) {
}()
edts := a.cfg.GetBool(cfgUploaderHeaderEnableDefaultTimestamp)
uploader := uploader.New(a.log, a.pool, edts)
downloader, err := downloader.New(ctx, a.log, a.pool)
downloader, err := downloader.New(a.log, downloader.Settings{ZipCompression: a.cfg.GetBool(cfgZipCompression)}, a.pool)
if err != nil {
a.log.Fatal("failed to create downloader", zap.Error(err))
}

View file

@ -231,12 +231,17 @@ func (o objectIDs) Slice() []string {
type Downloader struct {
log *zap.Logger
pool pool.Pool
settings Settings
}
type Settings struct {
ZipCompression bool
}
// New creates an instance of Downloader using specified options.
func New(ctx context.Context, log *zap.Logger, conns pool.Pool) (*Downloader, error) {
func New(log *zap.Logger, settings Settings, conns pool.Pool) (*Downloader, error) {
var err error
d := &Downloader{log: log, pool: conns}
d := &Downloader{log: log, pool: conns, settings: settings}
if err != nil {
return nil, fmt.Errorf("failed to get neofs client's reusable artifacts: %w", err)
}
@ -396,6 +401,11 @@ func (d *Downloader) DownloadZipped(c *fasthttp.RequestCtx) {
func (d *Downloader) streamFiles(c *fasthttp.RequestCtx, cid *cid.ID, ids []*object.ID) error {
zipWriter := zip.NewWriter(c)
compression := zip.Store
if d.settings.ZipCompression {
compression = zip.Deflate
}
for _, id := range ids {
var r io.Reader
readerInitCtx, initReader := context.WithCancel(c)
@ -413,7 +423,7 @@ func (d *Downloader) streamFiles(c *fasthttp.RequestCtx, cid *cid.ID, ids []*obj
header := &zip.FileHeader{
Name: getFilename(obj),
Method: zip.Store,
Method: compression,
Modified: time.Now(),
}
entryWriter, err := zipWriter.CreateHeader(header)

View file

@ -53,6 +53,9 @@ const (
// Peers.
cfgPeers = "peers"
// Zip compression.
cfgZipCompression = "zip.compression"
// Application.
cfgApplicationName = "app.name"
cfgApplicationVersion = "app.version"
@ -134,6 +137,9 @@ func settings() *viper.Viper {
// upload header
v.SetDefault(cfgUploaderHeaderEnableDefaultTimestamp, false)
// zip:
v.SetDefault(cfgZipCompression, false)
if err := v.BindPFlags(flags); err != nil {
panic(err)
}