From 0b364504a7edc6dce1dbc10be2ce7847f5747d57 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Tue, 19 Oct 2021 10:52:41 +0300 Subject: [PATCH] [#92] Support zip download Signed-off-by: Denis Kirillov --- app.go | 2 + downloader/download.go | 134 ++++++++++++++++++++++++++++++++++++++--- go.mod | 8 +-- go.sum | 25 +++++--- integration_test.go | 68 ++++++++++++++++++++- 5 files changed, 209 insertions(+), 28 deletions(-) diff --git a/app.go b/app.go index 52991ff..91c108a 100644 --- a/app.go +++ b/app.go @@ -205,6 +205,8 @@ func (a *app) Serve(ctx context.Context) { r.GET("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.logger(downloader.DownloadByAttribute)) r.HEAD("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.logger(downloader.HeadByAttribute)) a.log.Info("added path /get_by_attribute/{cid}/{attr_key}/{attr_val:*}") + r.GET("/zip/{cid}/{prefix:*}", a.logger(downloader.DownloadZipped)) + a.log.Info("added path /zip/{cid}/{prefix}") // enable metrics if a.cfg.GetBool(cmdMetrics) { a.log.Info("added path /metrics/") diff --git a/downloader/download.go b/downloader/download.go index cd5e87c..543e2f3 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -1,6 +1,7 @@ package downloader import ( + "archive/zip" "context" "errors" "fmt" @@ -308,9 +309,30 @@ func (d *Downloader) byAttribute(c *fasthttp.RequestCtx, f func(request, client. } func (d *Downloader) searchObject(c *fasthttp.RequestCtx, log *zap.Logger, cid *cid.ID, key, val string) (*object.Address, error) { + ids, err := d.searchByAttr(c, cid, key, val) + if err != nil { + return nil, err + } + if len(ids) > 1 { + log.Debug("found multiple objects", + zap.Strings("object_ids", objectIDs(ids).Slice()), + zap.Stringer("show_object_id", ids[0])) + } + + return formAddress(cid, ids[0]), nil +} + +func formAddress(cid *cid.ID, oid *object.ID) *object.Address { + address := object.NewAddress() + address.SetContainerID(cid) + address.SetObjectID(oid) + return address +} + +func (d *Downloader) search(c *fasthttp.RequestCtx, cid *cid.ID, key, val string, op object.SearchMatchType) ([]*object.ID, error) { options := object.NewSearchFilters() options.AddRootFilter() - options.AddFilter(key, val, object.MatchStringEqual) + options.AddFilter(key, val, op) sops := new(client.SearchObjectParams).WithContainerID(cid).WithSearchFilters(options) ids, err := d.pool.SearchObject(c, sops) @@ -320,13 +342,105 @@ func (d *Downloader) searchObject(c *fasthttp.RequestCtx, log *zap.Logger, cid * if len(ids) == 0 { return nil, errObjectNotFound } - if len(ids) > 1 { - log.Debug("found multiple objects", - zap.Strings("object_ids", objectIDs(ids).Slice()), - zap.Stringer("show_object_id", ids[0])) - } - address := object.NewAddress() - address.SetContainerID(cid) - address.SetObjectID(ids[0]) - return address, nil + return ids, nil +} + +func (d *Downloader) searchByPrefix(c *fasthttp.RequestCtx, cid *cid.ID, val string) ([]*object.ID, error) { + return d.search(c, cid, object.AttributeFileName, val, object.MatchCommonPrefix) +} + +func (d *Downloader) searchByAttr(c *fasthttp.RequestCtx, cid *cid.ID, key, val string) ([]*object.ID, error) { + return d.search(c, cid, key, val, object.MatchStringEqual) +} + +// DownloadZipped handles zip by prefix requests. +func (d *Downloader) DownloadZipped(c *fasthttp.RequestCtx) { + status := fasthttp.StatusBadRequest + scid, _ := c.UserValue("cid").(string) + prefix, _ := c.UserValue("prefix").(string) + log := d.log.With(zap.String("cid", scid), zap.String("prefix", prefix)) + + containerID := cid.New() + if err := containerID.Parse(scid); err != nil { + log.Error("wrong container id", zap.Error(err)) + c.Error("wrong container id", status) + return + } + + if err := tokens.StoreBearerToken(c); err != nil { + log.Error("could not fetch and store bearer token", zap.Error(err)) + c.Error("could not fetch and store bearer token", fasthttp.StatusBadRequest) + return + } + + ids, err := d.searchByPrefix(c, containerID, prefix) + if err != nil { + log.Error("couldn't find objects", zap.Error(err)) + if errors.Is(err, errObjectNotFound) { + status = fasthttp.StatusNotFound + } + c.Error("couldn't find objects", status) + return + } + + c.Response.Header.Set("Content-Type", "application/zip") + c.Response.Header.Set("Content-Disposition", "attachment; filename=\"archive.zip\"") + c.Response.SetStatusCode(http.StatusOK) + + if err = d.streamFiles(c, containerID, ids); err != nil { + log.Error("couldn't stream files", zap.Error(err)) + c.Error("couldn't stream", fasthttp.StatusInternalServerError) + return + } +} + +func (d *Downloader) streamFiles(c *fasthttp.RequestCtx, cid *cid.ID, ids []*object.ID) error { + zipWriter := zip.NewWriter(c) + for _, id := range ids { + var r io.Reader + readerInitCtx, initReader := context.WithCancel(c) + options := new(client.GetObjectParams). + WithAddress(formAddress(cid, id)). + WithPayloadReaderHandler(func(reader io.Reader) { + r = reader + initReader() + }) + + obj, err := d.pool.GetObject(c, options, bearerOpts(c)) + if err != nil { + return err + } + + header := &zip.FileHeader{ + Name: getFilename(obj), + Method: zip.Store, + Modified: time.Now(), + } + entryWriter, err := zipWriter.CreateHeader(header) + if err != nil { + return err + } + + <-readerInitCtx.Done() + _, err = io.Copy(entryWriter, r) + if err != nil { + return err + } + + if err = zipWriter.Flush(); err != nil { + return err + } + } + + return zipWriter.Close() +} + +func getFilename(obj *object.Object) string { + for _, attr := range obj.Attributes() { + if attr.Key() == object.AttributeFileName { + return attr.Value() + } + } + + return "" } diff --git a/go.mod b/go.mod index 4f7f1e7..094d8fd 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/google/uuid v1.3.0 // indirect github.com/klauspost/compress v1.13.1 // indirect github.com/nspcc-dev/neo-go v0.96.1 - github.com/nspcc-dev/neofs-api-go v1.28.3 + github.com/nspcc-dev/neofs-api-go v1.30.0 github.com/nspcc-dev/neofs-sdk-go v0.0.0-20210728093755-d95d722d6156 github.com/prometheus/client_golang v1.11.0 github.com/prometheus/common v0.29.0 @@ -27,9 +27,7 @@ require ( github.com/valyala/fasthttp v1.28.0 go.uber.org/multierr v1.7.0 // indirect go.uber.org/zap v1.18.1 - golang.org/x/net v0.0.0-20210716203947-853a461950ff // indirect - golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b // indirect - google.golang.org/genproto v0.0.0-20210722135532-667f2b7c528f // indirect - google.golang.org/grpc v1.39.0 + golang.org/x/tools v0.1.5 // indirect + google.golang.org/grpc v1.41.0 ) diff --git a/go.sum b/go.sum index e588d4b..669c380 100644 --- a/go.sum +++ b/go.sum @@ -165,6 +165,7 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE= github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU= @@ -309,6 +310,7 @@ github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5y github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= +github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= @@ -610,8 +612,9 @@ github.com/nspcc-dev/neo-go v0.96.1 h1:JaKWvM/vvQ48bq2ADNj7zH/6Ek38Iqxo22hdu2lhx github.com/nspcc-dev/neo-go v0.96.1/go.mod h1:yguwQBpWMTHx07INKoElJT8Gga1LUdTSi0gT75ywJ68= github.com/nspcc-dev/neofs-api-go v1.24.0/go.mod h1:G7dqincfdjBrAbL5nxVp82emF05fSVEqe59ICsoRDI8= github.com/nspcc-dev/neofs-api-go v1.27.1/go.mod h1:i0Cwgvcu9A4M4e58pydbXFisUhSxpfljmuWFPIp2btE= -github.com/nspcc-dev/neofs-api-go v1.28.3 h1:53Ec3hv3LtI3uuG1H8Yp2OKOIdsAqQVfUOyClhnhc9g= github.com/nspcc-dev/neofs-api-go v1.28.3/go.mod h1:YRIzUqBj/lGbmFm8mmCh54ZOzcJKkEIhv2s7ZvSLv3M= +github.com/nspcc-dev/neofs-api-go v1.30.0 h1:Nns7QjmQGk9I5lWMCtmeD9D3de46uyH3H07WBeXTEI0= +github.com/nspcc-dev/neofs-api-go v1.30.0/go.mod h1:KC8T91skIg8juvUh7lQabswQ9J6KmnXErpH8qwDitXA= github.com/nspcc-dev/neofs-crypto v0.2.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9KAEAUQR7dMxZmNA= github.com/nspcc-dev/neofs-crypto v0.2.3/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= github.com/nspcc-dev/neofs-crypto v0.3.0 h1:zlr3pgoxuzrmGCxc5W8dGVfA9Rro8diFvVnBg0L4ifM= @@ -982,8 +985,8 @@ golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLd golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210716203947-853a461950ff h1:j2EK/QoxYNBsXI4R7fQkkRUk8y6wnOBI+6hgPdP/6Ds= -golang.org/x/net v0.0.0-20210716203947-853a461950ff/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210928044308-7d9f5e0b762b h1:eB48h3HiRycXNy8E0Gf5e0hv7YT6Kt14L/D73G1fuwo= +golang.org/x/net v0.0.0-20210928044308-7d9f5e0b762b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1101,8 +1104,8 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6 h1:foEbQz/B0Oz6YIqu/69kfXPYeFQAuuMYFkjaqXzl5Wo= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210429154555-c04ba851c2a4/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -1115,8 +1118,9 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1264,8 +1268,8 @@ google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210722135532-667f2b7c528f h1:YORWxaStkWBnWgELOHTmDrqNlFXuVGEbhwbB5iK94bQ= -google.golang.org/genproto v0.0.0-20210722135532-667f2b7c528f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210928142010-c7af6a1a74c9 h1:XTH066D35LyHehRwlYhoK3qA+Hcgvg5xREG4kFQEW1Y= +google.golang.org/genproto v0.0.0-20210928142010-c7af6a1a74c9/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -1290,8 +1294,9 @@ google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.39.0 h1:Klz8I9kdtkIN6EpHHUOMLCYhTn/2WAe5a0s1hcBkdTI= -google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.41.0 h1:f+PlOh7QV4iIJkPrx5NQ7qaNGFQ3OTse67yaDHfju4E= +google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/integration_test.go b/integration_test.go index ab15cfc..2237e18 100644 --- a/integration_test.go +++ b/integration_test.go @@ -1,6 +1,7 @@ package main import ( + "archive/zip" "bytes" "context" "encoding/json" @@ -8,16 +9,16 @@ import ( "io" "mime/multipart" "net/http" + "sort" "strconv" "testing" "time" - "github.com/nspcc-dev/neofs-api-go/pkg/client" - "github.com/nspcc-dev/neofs-api-go/pkg/object" - "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "github.com/nspcc-dev/neofs-api-go/pkg/client" "github.com/nspcc-dev/neofs-api-go/pkg/container" cid "github.com/nspcc-dev/neofs-api-go/pkg/container/id" + "github.com/nspcc-dev/neofs-api-go/pkg/object" "github.com/nspcc-dev/neofs-sdk-go/pkg/policy" "github.com/nspcc-dev/neofs-sdk-go/pkg/pool" "github.com/spf13/viper" @@ -47,6 +48,7 @@ func TestIntegration(t *testing.T) { t.Run("simple put "+version, func(t *testing.T) { simplePut(ctx, t, clientPool, CID) }) t.Run("simple get "+version, func(t *testing.T) { simpleGet(ctx, t, clientPool, CID) }) t.Run("get by attribute "+version, func(t *testing.T) { getByAttr(ctx, t, clientPool, CID) }) + t.Run("get zip "+version, func(t *testing.T) { getZip(ctx, t, clientPool, CID) }) cancel() err = aioContainer.Terminate(ctx) @@ -174,6 +176,66 @@ func getByAttr(ctx context.Context, t *testing.T, clientPool pool.Pool, CID *cid } } +func getZip(ctx context.Context, t *testing.T, clientPool pool.Pool, CID *cid.ID) { + names := []string{"zipfolder/dir/name1.txt", "zipfolder/name2.txt"} + contents := []string{"content of file1", "content of file2"} + attributes1 := map[string]string{object.AttributeFileName: names[0]} + attributes2 := map[string]string{object.AttributeFileName: names[1]} + + putObject(ctx, t, clientPool, CID, contents[0], attributes1) + putObject(ctx, t, clientPool, CID, contents[1], attributes2) + + resp, err := http.Get("http://localhost:8082/zip/" + CID.String() + "/zipfolder") + require.NoError(t, err) + defer func() { + err = resp.Body.Close() + require.NoError(t, err) + }() + + data, err := io.ReadAll(resp.Body) + require.NoError(t, err) + checkZip(t, data, resp.ContentLength, names, contents) + + // check nested folder + resp2, err := http.Get("http://localhost:8082/zip/" + CID.String() + "/zipfolder/dir") + require.NoError(t, err) + defer func() { + err = resp2.Body.Close() + require.NoError(t, err) + }() + + data2, err := io.ReadAll(resp2.Body) + require.NoError(t, err) + checkZip(t, data2, resp2.ContentLength, names[:1], contents[:1]) +} + +func checkZip(t *testing.T, data []byte, length int64, names, contents []string) { + readerAt := bytes.NewReader(data) + + zipReader, err := zip.NewReader(readerAt, length) + require.NoError(t, err) + + require.Equal(t, len(names), len(zipReader.File)) + + sort.Slice(zipReader.File, func(i, j int) bool { + return zipReader.File[i].FileHeader.Name < zipReader.File[j].FileHeader.Name + }) + + for i, f := range zipReader.File { + require.Equal(t, names[i], f.FileHeader.Name) + + rc, err := f.Open() + require.NoError(t, err) + + all, err := io.ReadAll(rc) + require.NoError(t, err) + require.Equal(t, contents[i], string(all)) + + err = rc.Close() + require.NoError(t, err) + } +} + func createDockerContainer(ctx context.Context, t *testing.T, image string) testcontainers.Container { req := testcontainers.ContainerRequest{ Image: image,