forked from TrueCloudLab/frostfs-http-gw
[#96] Add zip compression
Signed-off-by: Denis Kirillov <denis@nspcc.ru>
This commit is contained in:
parent
62dd8b1927
commit
9b92c0146c
4 changed files with 35 additions and 6 deletions
13
README.md
13
README.md
|
@ -156,6 +156,10 @@ if needed.
|
||||||
All timing options accept values with suffixes, so "15s" is 15 seconds and
|
All timing options accept values with suffixes, so "15s" is 15 seconds and
|
||||||
"2m" is 2 minutes.
|
"2m" is 2 minutes.
|
||||||
|
|
||||||
|
### Zip streaming
|
||||||
|
The gateway supports downloading files by common prefix (like dir) in zip format. You can enable compression
|
||||||
|
using config or `HTTP_GW_ZIP_COMPRESSION=true` environment variable.
|
||||||
|
|
||||||
### Logging
|
### Logging
|
||||||
|
|
||||||
`--verbose` flag enables gRPC logging and there is a number of environment
|
`--verbose` flag enables gRPC logging and there is a number of environment
|
||||||
|
@ -192,6 +196,9 @@ peers:
|
||||||
0:
|
0:
|
||||||
address: grpc://s01.neofs.devenv:8080
|
address: grpc://s01.neofs.devenv:8080
|
||||||
weight: 1
|
weight: 1
|
||||||
|
|
||||||
|
zip:
|
||||||
|
compression: false
|
||||||
```
|
```
|
||||||
|
|
||||||
To know nesting level of variable you need to cut off the prefix `HTTP_GW` from variable and split the rest parts by `_`.
|
To know nesting level of variable you need to cut off the prefix `HTTP_GW` from variable and split the rest parts by `_`.
|
||||||
|
@ -311,6 +318,12 @@ $ wget http://localhost:8082/get/Dxhf4PNprrJHWWTG5RGLdfLkJiSQ3AQqit1MSnEPRkDZ/2m
|
||||||
|
|
||||||
```
|
```
|
||||||
|
|
||||||
|
##### Zip
|
||||||
|
You can download some dir (files with the same prefix) in zip (it will be compressed if config contains appropriate param):
|
||||||
|
```
|
||||||
|
$ wget http://localhost:8082/zip/Dxhf4PNprrJHWWTG5RGLdfLkJiSQ3AQqit1MSnEPRkDZ/common/prefix
|
||||||
|
```
|
||||||
|
|
||||||
#### Replies
|
#### Replies
|
||||||
|
|
||||||
You get object contents in the reply body (if GET method was used), but at the same time you also get a
|
You get object contents in the reply body (if GET method was used), but at the same time you also get a
|
||||||
|
|
2
app.go
2
app.go
|
@ -190,7 +190,7 @@ func (a *app) Serve(ctx context.Context) {
|
||||||
}()
|
}()
|
||||||
edts := a.cfg.GetBool(cfgUploaderHeaderEnableDefaultTimestamp)
|
edts := a.cfg.GetBool(cfgUploaderHeaderEnableDefaultTimestamp)
|
||||||
uploader := uploader.New(a.log, a.pool, edts)
|
uploader := uploader.New(a.log, a.pool, edts)
|
||||||
downloader, err := downloader.New(ctx, a.log, a.pool)
|
downloader, err := downloader.New(a.log, downloader.Settings{ZipCompression: a.cfg.GetBool(cfgZipCompression)}, a.pool)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
a.log.Fatal("failed to create downloader", zap.Error(err))
|
a.log.Fatal("failed to create downloader", zap.Error(err))
|
||||||
}
|
}
|
||||||
|
|
|
@ -231,12 +231,17 @@ func (o objectIDs) Slice() []string {
|
||||||
type Downloader struct {
|
type Downloader struct {
|
||||||
log *zap.Logger
|
log *zap.Logger
|
||||||
pool pool.Pool
|
pool pool.Pool
|
||||||
|
settings Settings
|
||||||
|
}
|
||||||
|
|
||||||
|
type Settings struct {
|
||||||
|
ZipCompression bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// New creates an instance of Downloader using specified options.
|
// New creates an instance of Downloader using specified options.
|
||||||
func New(ctx context.Context, log *zap.Logger, conns pool.Pool) (*Downloader, error) {
|
func New(log *zap.Logger, settings Settings, conns pool.Pool) (*Downloader, error) {
|
||||||
var err error
|
var err error
|
||||||
d := &Downloader{log: log, pool: conns}
|
d := &Downloader{log: log, pool: conns, settings: settings}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to get neofs client's reusable artifacts: %w", err)
|
return nil, fmt.Errorf("failed to get neofs client's reusable artifacts: %w", err)
|
||||||
}
|
}
|
||||||
|
@ -396,6 +401,11 @@ func (d *Downloader) DownloadZipped(c *fasthttp.RequestCtx) {
|
||||||
|
|
||||||
func (d *Downloader) streamFiles(c *fasthttp.RequestCtx, cid *cid.ID, ids []*object.ID) error {
|
func (d *Downloader) streamFiles(c *fasthttp.RequestCtx, cid *cid.ID, ids []*object.ID) error {
|
||||||
zipWriter := zip.NewWriter(c)
|
zipWriter := zip.NewWriter(c)
|
||||||
|
compression := zip.Store
|
||||||
|
if d.settings.ZipCompression {
|
||||||
|
compression = zip.Deflate
|
||||||
|
}
|
||||||
|
|
||||||
for _, id := range ids {
|
for _, id := range ids {
|
||||||
var r io.Reader
|
var r io.Reader
|
||||||
readerInitCtx, initReader := context.WithCancel(c)
|
readerInitCtx, initReader := context.WithCancel(c)
|
||||||
|
@ -413,7 +423,7 @@ func (d *Downloader) streamFiles(c *fasthttp.RequestCtx, cid *cid.ID, ids []*obj
|
||||||
|
|
||||||
header := &zip.FileHeader{
|
header := &zip.FileHeader{
|
||||||
Name: getFilename(obj),
|
Name: getFilename(obj),
|
||||||
Method: zip.Store,
|
Method: compression,
|
||||||
Modified: time.Now(),
|
Modified: time.Now(),
|
||||||
}
|
}
|
||||||
entryWriter, err := zipWriter.CreateHeader(header)
|
entryWriter, err := zipWriter.CreateHeader(header)
|
||||||
|
|
|
@ -53,6 +53,9 @@ const (
|
||||||
// Peers.
|
// Peers.
|
||||||
cfgPeers = "peers"
|
cfgPeers = "peers"
|
||||||
|
|
||||||
|
// Zip compression.
|
||||||
|
cfgZipCompression = "zip.compression"
|
||||||
|
|
||||||
// Application.
|
// Application.
|
||||||
cfgApplicationName = "app.name"
|
cfgApplicationName = "app.name"
|
||||||
cfgApplicationVersion = "app.version"
|
cfgApplicationVersion = "app.version"
|
||||||
|
@ -134,6 +137,9 @@ func settings() *viper.Viper {
|
||||||
// upload header
|
// upload header
|
||||||
v.SetDefault(cfgUploaderHeaderEnableDefaultTimestamp, false)
|
v.SetDefault(cfgUploaderHeaderEnableDefaultTimestamp, false)
|
||||||
|
|
||||||
|
// zip:
|
||||||
|
v.SetDefault(cfgZipCompression, false)
|
||||||
|
|
||||||
if err := v.BindPFlags(flags); err != nil {
|
if err := v.BindPFlags(flags); err != nil {
|
||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue