[#92] Support zip download
Signed-off-by: Denis Kirillov <denis@nspcc.ru>
This commit is contained in:
Denis Kirillov
Alex Vanin
parent
0e2861152d
commit
0b364504a7
5 changed files with 209 additions and 28 deletions
2
app.go
2
app.go
|
|
@ -205,6 +205,8 @@ func (a *app) Serve(ctx context.Context) {
|
|||
r.GET("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.logger(downloader.DownloadByAttribute))
|
||||
r.HEAD("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.logger(downloader.HeadByAttribute))
|
||||
a.log.Info("added path /get_by_attribute/{cid}/{attr_key}/{attr_val:*}")
|
||||
r.GET("/zip/{cid}/{prefix:*}", a.logger(downloader.DownloadZipped))
|
||||
a.log.Info("added path /zip/{cid}/{prefix}")
|
||||
// enable metrics
|
||||
if a.cfg.GetBool(cmdMetrics) {
|
||||
a.log.Info("added path /metrics/")
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
package downloader
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
|
@ -308,9 +309,30 @@ func (d *Downloader) byAttribute(c *fasthttp.RequestCtx, f func(request, client.
|
|||
}
|
||||
|
||||
func (d *Downloader) searchObject(c *fasthttp.RequestCtx, log *zap.Logger, cid *cid.ID, key, val string) (*object.Address, error) {
|
||||
ids, err := d.searchByAttr(c, cid, key, val)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(ids) > 1 {
|
||||
log.Debug("found multiple objects",
|
||||
zap.Strings("object_ids", objectIDs(ids).Slice()),
|
||||
zap.Stringer("show_object_id", ids[0]))
|
||||
}
|
||||
|
||||
return formAddress(cid, ids[0]), nil
|
||||
}
|
||||
|
||||
func formAddress(cid *cid.ID, oid *object.ID) *object.Address {
|
||||
address := object.NewAddress()
|
||||
address.SetContainerID(cid)
|
||||
address.SetObjectID(oid)
|
||||
return address
|
||||
}
|
||||
|
||||
func (d *Downloader) search(c *fasthttp.RequestCtx, cid *cid.ID, key, val string, op object.SearchMatchType) ([]*object.ID, error) {
|
||||
options := object.NewSearchFilters()
|
||||
options.AddRootFilter()
|
||||
options.AddFilter(key, val, object.MatchStringEqual)
|
||||
options.AddFilter(key, val, op)
|
||||
|
||||
sops := new(client.SearchObjectParams).WithContainerID(cid).WithSearchFilters(options)
|
||||
ids, err := d.pool.SearchObject(c, sops)
|
||||
|
|
@ -320,13 +342,105 @@ func (d *Downloader) searchObject(c *fasthttp.RequestCtx, log *zap.Logger, cid *
|
|||
if len(ids) == 0 {
|
||||
return nil, errObjectNotFound
|
||||
}
|
||||
if len(ids) > 1 {
|
||||
log.Debug("found multiple objects",
|
||||
zap.Strings("object_ids", objectIDs(ids).Slice()),
|
||||
zap.Stringer("show_object_id", ids[0]))
|
||||
}
|
||||
address := object.NewAddress()
|
||||
address.SetContainerID(cid)
|
||||
address.SetObjectID(ids[0])
|
||||
return address, nil
|
||||
return ids, nil
|
||||
}
|
||||
|
||||
func (d *Downloader) searchByPrefix(c *fasthttp.RequestCtx, cid *cid.ID, val string) ([]*object.ID, error) {
|
||||
return d.search(c, cid, object.AttributeFileName, val, object.MatchCommonPrefix)
|
||||
}
|
||||
|
||||
func (d *Downloader) searchByAttr(c *fasthttp.RequestCtx, cid *cid.ID, key, val string) ([]*object.ID, error) {
|
||||
return d.search(c, cid, key, val, object.MatchStringEqual)
|
||||
}
|
||||
|
||||
// DownloadZipped handles zip by prefix requests.
|
||||
func (d *Downloader) DownloadZipped(c *fasthttp.RequestCtx) {
|
||||
status := fasthttp.StatusBadRequest
|
||||
scid, _ := c.UserValue("cid").(string)
|
||||
prefix, _ := c.UserValue("prefix").(string)
|
||||
log := d.log.With(zap.String("cid", scid), zap.String("prefix", prefix))
|
||||
|
||||
containerID := cid.New()
|
||||
if err := containerID.Parse(scid); err != nil {
|
||||
log.Error("wrong container id", zap.Error(err))
|
||||
c.Error("wrong container id", status)
|
||||
return
|
||||
}
|
||||
|
||||
if err := tokens.StoreBearerToken(c); err != nil {
|
||||
log.Error("could not fetch and store bearer token", zap.Error(err))
|
||||
c.Error("could not fetch and store bearer token", fasthttp.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
ids, err := d.searchByPrefix(c, containerID, prefix)
|
||||
if err != nil {
|
||||
log.Error("couldn't find objects", zap.Error(err))
|
||||
if errors.Is(err, errObjectNotFound) {
|
||||
status = fasthttp.StatusNotFound
|
||||
}
|
||||
c.Error("couldn't find objects", status)
|
||||
return
|
||||
}
|
||||
|
||||
c.Response.Header.Set("Content-Type", "application/zip")
|
||||
c.Response.Header.Set("Content-Disposition", "attachment; filename=\"archive.zip\"")
|
||||
c.Response.SetStatusCode(http.StatusOK)
|
||||
|
||||
if err = d.streamFiles(c, containerID, ids); err != nil {
|
||||
log.Error("couldn't stream files", zap.Error(err))
|
||||
c.Error("couldn't stream", fasthttp.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Downloader) streamFiles(c *fasthttp.RequestCtx, cid *cid.ID, ids []*object.ID) error {
|
||||
zipWriter := zip.NewWriter(c)
|
||||
for _, id := range ids {
|
||||
var r io.Reader
|
||||
readerInitCtx, initReader := context.WithCancel(c)
|
||||
options := new(client.GetObjectParams).
|
||||
WithAddress(formAddress(cid, id)).
|
||||
WithPayloadReaderHandler(func(reader io.Reader) {
|
||||
r = reader
|
||||
initReader()
|
||||
})
|
||||
|
||||
obj, err := d.pool.GetObject(c, options, bearerOpts(c))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
header := &zip.FileHeader{
|
||||
Name: getFilename(obj),
|
||||
Method: zip.Store,
|
||||
Modified: time.Now(),
|
||||
}
|
||||
entryWriter, err := zipWriter.CreateHeader(header)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
<-readerInitCtx.Done()
|
||||
_, err = io.Copy(entryWriter, r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err = zipWriter.Flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return zipWriter.Close()
|
||||
}
|
||||
|
||||
func getFilename(obj *object.Object) string {
|
||||
for _, attr := range obj.Attributes() {
|
||||
if attr.Key() == object.AttributeFileName {
|
||||
return attr.Value()
|
||||
}
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
|
|
|||
8
go.mod
8
go.mod
|
|
@ -15,7 +15,7 @@ require (
|
|||
github.com/google/uuid v1.3.0 // indirect
|
||||
github.com/klauspost/compress v1.13.1 // indirect
|
||||
github.com/nspcc-dev/neo-go v0.96.1
|
||||
github.com/nspcc-dev/neofs-api-go v1.28.3
|
||||
github.com/nspcc-dev/neofs-api-go v1.30.0
|
||||
github.com/nspcc-dev/neofs-sdk-go v0.0.0-20210728093755-d95d722d6156
|
||||
github.com/prometheus/client_golang v1.11.0
|
||||
github.com/prometheus/common v0.29.0
|
||||
|
|
@ -27,9 +27,7 @@ require (
|
|||
github.com/valyala/fasthttp v1.28.0
|
||||
go.uber.org/multierr v1.7.0 // indirect
|
||||
go.uber.org/zap v1.18.1
|
||||
golang.org/x/net v0.0.0-20210716203947-853a461950ff // indirect
|
||||
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect
|
||||
golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b // indirect
|
||||
google.golang.org/genproto v0.0.0-20210722135532-667f2b7c528f // indirect
|
||||
google.golang.org/grpc v1.39.0
|
||||
golang.org/x/tools v0.1.5 // indirect
|
||||
google.golang.org/grpc v1.41.0
|
||||
)
|
||||
|
|
|
|||
25
go.sum
25
go.sum
|
|
@ -165,6 +165,7 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX
|
|||
github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
|
||||
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
|
||||
github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
|
||||
github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
|
||||
github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
|
||||
github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE=
|
||||
github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU=
|
||||
|
|
@ -309,6 +310,7 @@ github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5y
|
|||
github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
|
||||
github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
|
||||
github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=
|
||||
github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
|
||||
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
|
||||
github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw=
|
||||
github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
|
||||
|
|
@ -610,8 +612,9 @@ github.com/nspcc-dev/neo-go v0.96.1 h1:JaKWvM/vvQ48bq2ADNj7zH/6Ek38Iqxo22hdu2lhx
|
|||
github.com/nspcc-dev/neo-go v0.96.1/go.mod h1:yguwQBpWMTHx07INKoElJT8Gga1LUdTSi0gT75ywJ68=
|
||||
github.com/nspcc-dev/neofs-api-go v1.24.0/go.mod h1:G7dqincfdjBrAbL5nxVp82emF05fSVEqe59ICsoRDI8=
|
||||
github.com/nspcc-dev/neofs-api-go v1.27.1/go.mod h1:i0Cwgvcu9A4M4e58pydbXFisUhSxpfljmuWFPIp2btE=
|
||||
github.com/nspcc-dev/neofs-api-go v1.28.3 h1:53Ec3hv3LtI3uuG1H8Yp2OKOIdsAqQVfUOyClhnhc9g=
|
||||
github.com/nspcc-dev/neofs-api-go v1.28.3/go.mod h1:YRIzUqBj/lGbmFm8mmCh54ZOzcJKkEIhv2s7ZvSLv3M=
|
||||
github.com/nspcc-dev/neofs-api-go v1.30.0 h1:Nns7QjmQGk9I5lWMCtmeD9D3de46uyH3H07WBeXTEI0=
|
||||
github.com/nspcc-dev/neofs-api-go v1.30.0/go.mod h1:KC8T91skIg8juvUh7lQabswQ9J6KmnXErpH8qwDitXA=
|
||||
github.com/nspcc-dev/neofs-crypto v0.2.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9KAEAUQR7dMxZmNA=
|
||||
github.com/nspcc-dev/neofs-crypto v0.2.3/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw=
|
||||
github.com/nspcc-dev/neofs-crypto v0.3.0 h1:zlr3pgoxuzrmGCxc5W8dGVfA9Rro8diFvVnBg0L4ifM=
|
||||
|
|
@ -982,8 +985,8 @@ golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLd
|
|||
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
|
||||
golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20210716203947-853a461950ff h1:j2EK/QoxYNBsXI4R7fQkkRUk8y6wnOBI+6hgPdP/6Ds=
|
||||
golang.org/x/net v0.0.0-20210716203947-853a461950ff/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20210928044308-7d9f5e0b762b h1:eB48h3HiRycXNy8E0Gf5e0hv7YT6Kt14L/D73G1fuwo=
|
||||
golang.org/x/net v0.0.0-20210928044308-7d9f5e0b762b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
|
|
@ -1101,8 +1104,8 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc
|
|||
golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I=
|
||||
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6 h1:foEbQz/B0Oz6YIqu/69kfXPYeFQAuuMYFkjaqXzl5Wo=
|
||||
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210429154555-c04ba851c2a4/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
|
|
@ -1115,8 +1118,9 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
|||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
|
||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
|
|
@ -1264,8 +1268,8 @@ google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6D
|
|||
google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A=
|
||||
google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
|
||||
google.golang.org/genproto v0.0.0-20210722135532-667f2b7c528f h1:YORWxaStkWBnWgELOHTmDrqNlFXuVGEbhwbB5iK94bQ=
|
||||
google.golang.org/genproto v0.0.0-20210722135532-667f2b7c528f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48=
|
||||
google.golang.org/genproto v0.0.0-20210928142010-c7af6a1a74c9 h1:XTH066D35LyHehRwlYhoK3qA+Hcgvg5xREG4kFQEW1Y=
|
||||
google.golang.org/genproto v0.0.0-20210928142010-c7af6a1a74c9/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
|
||||
google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
|
||||
|
|
@ -1290,8 +1294,9 @@ google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG
|
|||
google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
|
||||
google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
|
||||
google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
|
||||
google.golang.org/grpc v1.39.0 h1:Klz8I9kdtkIN6EpHHUOMLCYhTn/2WAe5a0s1hcBkdTI=
|
||||
google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE=
|
||||
google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
|
||||
google.golang.org/grpc v1.41.0 h1:f+PlOh7QV4iIJkPrx5NQ7qaNGFQ3OTse67yaDHfju4E=
|
||||
google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k=
|
||||
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
|
||||
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
|
||||
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
|
|
@ -8,16 +9,16 @@ import (
|
|||
"io"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nspcc-dev/neofs-api-go/pkg/client"
|
||||
"github.com/nspcc-dev/neofs-api-go/pkg/object"
|
||||
|
||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||
"github.com/nspcc-dev/neofs-api-go/pkg/client"
|
||||
"github.com/nspcc-dev/neofs-api-go/pkg/container"
|
||||
cid "github.com/nspcc-dev/neofs-api-go/pkg/container/id"
|
||||
"github.com/nspcc-dev/neofs-api-go/pkg/object"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/pkg/policy"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/pkg/pool"
|
||||
"github.com/spf13/viper"
|
||||
|
|
@ -47,6 +48,7 @@ func TestIntegration(t *testing.T) {
|
|||
t.Run("simple put "+version, func(t *testing.T) { simplePut(ctx, t, clientPool, CID) })
|
||||
t.Run("simple get "+version, func(t *testing.T) { simpleGet(ctx, t, clientPool, CID) })
|
||||
t.Run("get by attribute "+version, func(t *testing.T) { getByAttr(ctx, t, clientPool, CID) })
|
||||
t.Run("get zip "+version, func(t *testing.T) { getZip(ctx, t, clientPool, CID) })
|
||||
|
||||
cancel()
|
||||
err = aioContainer.Terminate(ctx)
|
||||
|
|
@ -174,6 +176,66 @@ func getByAttr(ctx context.Context, t *testing.T, clientPool pool.Pool, CID *cid
|
|||
}
|
||||
}
|
||||
|
||||
func getZip(ctx context.Context, t *testing.T, clientPool pool.Pool, CID *cid.ID) {
|
||||
names := []string{"zipfolder/dir/name1.txt", "zipfolder/name2.txt"}
|
||||
contents := []string{"content of file1", "content of file2"}
|
||||
attributes1 := map[string]string{object.AttributeFileName: names[0]}
|
||||
attributes2 := map[string]string{object.AttributeFileName: names[1]}
|
||||
|
||||
putObject(ctx, t, clientPool, CID, contents[0], attributes1)
|
||||
putObject(ctx, t, clientPool, CID, contents[1], attributes2)
|
||||
|
||||
resp, err := http.Get("http://localhost:8082/zip/" + CID.String() + "/zipfolder")
|
||||
require.NoError(t, err)
|
||||
defer func() {
|
||||
err = resp.Body.Close()
|
||||
require.NoError(t, err)
|
||||
}()
|
||||
|
||||
data, err := io.ReadAll(resp.Body)
|
||||
require.NoError(t, err)
|
||||
checkZip(t, data, resp.ContentLength, names, contents)
|
||||
|
||||
// check nested folder
|
||||
resp2, err := http.Get("http://localhost:8082/zip/" + CID.String() + "/zipfolder/dir")
|
||||
require.NoError(t, err)
|
||||
defer func() {
|
||||
err = resp2.Body.Close()
|
||||
require.NoError(t, err)
|
||||
}()
|
||||
|
||||
data2, err := io.ReadAll(resp2.Body)
|
||||
require.NoError(t, err)
|
||||
checkZip(t, data2, resp2.ContentLength, names[:1], contents[:1])
|
||||
}
|
||||
|
||||
func checkZip(t *testing.T, data []byte, length int64, names, contents []string) {
|
||||
readerAt := bytes.NewReader(data)
|
||||
|
||||
zipReader, err := zip.NewReader(readerAt, length)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Equal(t, len(names), len(zipReader.File))
|
||||
|
||||
sort.Slice(zipReader.File, func(i, j int) bool {
|
||||
return zipReader.File[i].FileHeader.Name < zipReader.File[j].FileHeader.Name
|
||||
})
|
||||
|
||||
for i, f := range zipReader.File {
|
||||
require.Equal(t, names[i], f.FileHeader.Name)
|
||||
|
||||
rc, err := f.Open()
|
||||
require.NoError(t, err)
|
||||
|
||||
all, err := io.ReadAll(rc)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, contents[i], string(all))
|
||||
|
||||
err = rc.Close()
|
||||
require.NoError(t, err)
|
||||
}
|
||||
}
|
||||
|
||||
func createDockerContainer(ctx context.Context, t *testing.T, image string) testcontainers.Container {
|
||||
req := testcontainers.ContainerRequest{
|
||||
Image: image,
|
||||
|
|
|
|||
Loading…
Reference in a new issue