[#96] Add zip compression

Signed-off-by: Denis Kirillov <denis@nspcc.ru>
This commit is contained in:
Denis Kirillov 2021-10-29 16:11:34 +03:00 committed by Alex Vanin
parent 62dd8b1927
commit 9b92c0146c
4 changed files with 35 additions and 6 deletions

View file

@ -156,6 +156,10 @@ if needed.
All timing options accept values with suffixes, so "15s" is 15 seconds and All timing options accept values with suffixes, so "15s" is 15 seconds and
"2m" is 2 minutes. "2m" is 2 minutes.
### Zip streaming
The gateway supports downloading files by common prefix (like dir) in zip format. You can enable compression
using config or `HTTP_GW_ZIP_COMPRESSION=true` environment variable.
### Logging ### Logging
`--verbose` flag enables gRPC logging and there is a number of environment `--verbose` flag enables gRPC logging and there is a number of environment
@ -192,6 +196,9 @@ peers:
0: 0:
address: grpc://s01.neofs.devenv:8080 address: grpc://s01.neofs.devenv:8080
weight: 1 weight: 1
zip:
compression: false
``` ```
To know nesting level of variable you need to cut off the prefix `HTTP_GW` from variable and split the rest parts by `_`. To know nesting level of variable you need to cut off the prefix `HTTP_GW` from variable and split the rest parts by `_`.
@ -311,6 +318,12 @@ $ wget http://localhost:8082/get/Dxhf4PNprrJHWWTG5RGLdfLkJiSQ3AQqit1MSnEPRkDZ/2m
``` ```
##### Zip
You can download some dir (files with the same prefix) in zip (it will be compressed if config contains appropriate param):
```
$ wget http://localhost:8082/zip/Dxhf4PNprrJHWWTG5RGLdfLkJiSQ3AQqit1MSnEPRkDZ/common/prefix
```
#### Replies #### Replies
You get object contents in the reply body (if GET method was used), but at the same time you also get a You get object contents in the reply body (if GET method was used), but at the same time you also get a

2
app.go
View file

@ -190,7 +190,7 @@ func (a *app) Serve(ctx context.Context) {
}() }()
edts := a.cfg.GetBool(cfgUploaderHeaderEnableDefaultTimestamp) edts := a.cfg.GetBool(cfgUploaderHeaderEnableDefaultTimestamp)
uploader := uploader.New(a.log, a.pool, edts) uploader := uploader.New(a.log, a.pool, edts)
downloader, err := downloader.New(ctx, a.log, a.pool) downloader, err := downloader.New(a.log, downloader.Settings{ZipCompression: a.cfg.GetBool(cfgZipCompression)}, a.pool)
if err != nil { if err != nil {
a.log.Fatal("failed to create downloader", zap.Error(err)) a.log.Fatal("failed to create downloader", zap.Error(err))
} }

View file

@ -231,12 +231,17 @@ func (o objectIDs) Slice() []string {
type Downloader struct { type Downloader struct {
log *zap.Logger log *zap.Logger
pool pool.Pool pool pool.Pool
settings Settings
}
type Settings struct {
ZipCompression bool
} }
// New creates an instance of Downloader using specified options. // New creates an instance of Downloader using specified options.
func New(ctx context.Context, log *zap.Logger, conns pool.Pool) (*Downloader, error) { func New(log *zap.Logger, settings Settings, conns pool.Pool) (*Downloader, error) {
var err error var err error
d := &Downloader{log: log, pool: conns} d := &Downloader{log: log, pool: conns, settings: settings}
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to get neofs client's reusable artifacts: %w", err) return nil, fmt.Errorf("failed to get neofs client's reusable artifacts: %w", err)
} }
@ -396,6 +401,11 @@ func (d *Downloader) DownloadZipped(c *fasthttp.RequestCtx) {
func (d *Downloader) streamFiles(c *fasthttp.RequestCtx, cid *cid.ID, ids []*object.ID) error { func (d *Downloader) streamFiles(c *fasthttp.RequestCtx, cid *cid.ID, ids []*object.ID) error {
zipWriter := zip.NewWriter(c) zipWriter := zip.NewWriter(c)
compression := zip.Store
if d.settings.ZipCompression {
compression = zip.Deflate
}
for _, id := range ids { for _, id := range ids {
var r io.Reader var r io.Reader
readerInitCtx, initReader := context.WithCancel(c) readerInitCtx, initReader := context.WithCancel(c)
@ -413,7 +423,7 @@ func (d *Downloader) streamFiles(c *fasthttp.RequestCtx, cid *cid.ID, ids []*obj
header := &zip.FileHeader{ header := &zip.FileHeader{
Name: getFilename(obj), Name: getFilename(obj),
Method: zip.Store, Method: compression,
Modified: time.Now(), Modified: time.Now(),
} }
entryWriter, err := zipWriter.CreateHeader(header) entryWriter, err := zipWriter.CreateHeader(header)

View file

@ -53,6 +53,9 @@ const (
// Peers. // Peers.
cfgPeers = "peers" cfgPeers = "peers"
// Zip compression.
cfgZipCompression = "zip.compression"
// Application. // Application.
cfgApplicationName = "app.name" cfgApplicationName = "app.name"
cfgApplicationVersion = "app.version" cfgApplicationVersion = "app.version"
@ -134,6 +137,9 @@ func settings() *viper.Viper {
// upload header // upload header
v.SetDefault(cfgUploaderHeaderEnableDefaultTimestamp, false) v.SetDefault(cfgUploaderHeaderEnableDefaultTimestamp, false)
// zip:
v.SetDefault(cfgZipCompression, false)
if err := v.BindPFlags(flags); err != nil { if err := v.BindPFlags(flags); err != nil {
panic(err) panic(err)
} }