From 2b7e4a36fbc40cb5f87b789b7895bca12e6c182c Mon Sep 17 00:00:00 2001 From: Leonard Lyubich Date: Tue, 8 Feb 2022 19:24:23 +0300 Subject: [PATCH] [#126] Upgrade NeoFS SDK Go library Core changes: - `object.ID` moved to new package `oid`; - `object.Address` moved to new package `address`; - `pool.Object` interface changes. Signed-off-by: Leonard Lyubich --- app.go | 2 - downloader/download.go | 262 +++++++++++++++++++++-------------------- downloader/head.go | 31 +++-- go.mod | 12 +- go.sum | 30 +++-- integration_test.go | 43 +++---- uploader/upload.go | 26 ++-- 7 files changed, 220 insertions(+), 186 deletions(-) diff --git a/app.go b/app.go index 267c92c..294c306 100644 --- a/app.go +++ b/app.go @@ -4,7 +4,6 @@ import ( "context" "crypto/ecdsa" "fmt" - "math" "strconv" "github.com/fasthttp/router" @@ -122,7 +121,6 @@ func newApp(ctx context.Context, opt ...Option) App { NodeConnectionTimeout: a.cfg.GetDuration(cfgConTimeout), NodeRequestTimeout: a.cfg.GetDuration(cfgReqTimeout), ClientRebalanceInterval: a.cfg.GetDuration(cfgRebalance), - SessionExpirationEpoch: math.MaxUint64, } a.pool, err = pb.Build(ctx, opts) if err != nil { diff --git a/downloader/download.go b/downloader/download.go index 4964eba..89ebb43 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -16,9 +16,10 @@ import ( "github.com/nspcc-dev/neofs-http-gw/response" "github.com/nspcc-dev/neofs-http-gw/tokens" "github.com/nspcc-dev/neofs-http-gw/utils" - "github.com/nspcc-dev/neofs-sdk-go/client" cid "github.com/nspcc-dev/neofs-sdk-go/container/id" "github.com/nspcc-dev/neofs-sdk-go/object" + "github.com/nspcc-dev/neofs-sdk-go/object/address" + oid "github.com/nspcc-dev/neofs-sdk-go/object/id" "github.com/nspcc-dev/neofs-sdk-go/pool" "github.com/valyala/fasthttp" "go.uber.org/zap" @@ -38,8 +39,6 @@ type ( log *zap.Logger } - objectIDs []*object.ID - errReader struct { data []byte err error @@ -116,39 +115,39 @@ func isValidValue(s string) bool { return true } -func (r request) receiveFile(clnt pool.Object, objectAddress *object.Address) { +func (r request) receiveFile(clnt pool.Object, objectAddress *address.Address) { var ( err error dis = "inline" start = time.Now() filename string - obj *object.Object ) if err = tokens.StoreBearerToken(r.RequestCtx); err != nil { r.log.Error("could not fetch and store bearer token", zap.Error(err)) response.Error(r.RequestCtx, "could not fetch and store bearer token", fasthttp.StatusBadRequest) return } - readDetector := newDetector() - options := new(client.GetObjectParams). - WithAddress(objectAddress). - WithPayloadReaderHandler(func(reader io.Reader) { - readDetector.SetReader(reader) - readDetector.Detect() - }) - obj, err = clnt.GetObject(r.RequestCtx, options, bearerOpts(r.RequestCtx)) + rObj, err := clnt.GetObject(r.RequestCtx, *objectAddress, bearerOpts(r.RequestCtx)) if err != nil { r.handleNeoFSErr(err, start) return } + + // we can't close reader in this function, so how to do it? + if r.Request.URI().QueryArgs().GetBool("download") { dis = "attachment" } - r.Response.SetBodyStream(readDetector.MultiReader(), int(obj.PayloadSize())) - r.Response.Header.Set(fasthttp.HeaderContentLength, strconv.FormatUint(obj.PayloadSize(), 10)) + + readDetector := newDetector() + readDetector.SetReader(rObj.Payload) + readDetector.Detect() + + r.Response.SetBodyStream(readDetector.MultiReader(), int(rObj.Header.PayloadSize())) + r.Response.Header.Set(fasthttp.HeaderContentLength, strconv.FormatUint(rObj.Header.PayloadSize(), 10)) var contentType string - for _, attr := range obj.Attributes() { + for _, attr := range rObj.Header.Attributes() { key := attr.Key() val := attr.Value() if !isValidToken(key) || !isValidValue(val) { @@ -177,7 +176,7 @@ func (r request) receiveFile(clnt pool.Object, objectAddress *object.Address) { } } - idsToResponse(&r.Response, obj) + idsToResponse(&r.Response, &rObj.Header) if len(contentType) == 0 { if readDetector.err != nil { @@ -244,14 +243,6 @@ func (r *request) handleNeoFSErr(err error, start time.Time) { response.Error(r.RequestCtx, msg, code) } -func (o objectIDs) Slice() []string { - res := make([]string, 0, len(o)) - for _, oid := range o { - res = append(res, oid.String()) - } - return res -} - // Downloader is a download request handler. type Downloader struct { log *zap.Logger @@ -287,21 +278,21 @@ func (d *Downloader) DownloadByAddress(c *fasthttp.RequestCtx) { // byAddress is wrapper for function (e.g. request.headObject, request.receiveFile) that // prepares request and object address to it. -func (d *Downloader) byAddress(c *fasthttp.RequestCtx, f func(request, pool.Object, *object.Address)) { +func (d *Downloader) byAddress(c *fasthttp.RequestCtx, f func(request, pool.Object, *address.Address)) { var ( - address = object.NewAddress() - cid, _ = c.UserValue("cid").(string) - oid, _ = c.UserValue("oid").(string) - val = strings.Join([]string{cid, oid}, "/") - log = d.log.With(zap.String("cid", cid), zap.String("oid", oid)) + addr = address.NewAddress() + idCnr, _ = c.UserValue("cid").(string) + idObj, _ = c.UserValue("oid").(string) + val = strings.Join([]string{idCnr, idObj}, "/") + log = d.log.With(zap.String("cid", idCnr), zap.String("oid", idObj)) ) - if err := address.Parse(val); err != nil { + if err := addr.Parse(val); err != nil { log.Error("wrong object address", zap.Error(err)) response.Error(c, "wrong object address", fasthttp.StatusBadRequest) return } - f(*d.newRequest(c, log), d.pool, address) + f(*d.newRequest(c, log), d.pool, addr) } // DownloadByAttribute handles attribute-based download requests. @@ -310,7 +301,7 @@ func (d *Downloader) DownloadByAttribute(c *fasthttp.RequestCtx) { } // byAttribute is wrapper similar to byAddress. -func (d *Downloader) byAttribute(c *fasthttp.RequestCtx, f func(request, pool.Object, *object.Address)) { +func (d *Downloader) byAttribute(c *fasthttp.RequestCtx, f func(request, pool.Object, *address.Address)) { var ( httpStatus = fasthttp.StatusBadRequest scid, _ = c.UserValue("cid").(string) @@ -325,67 +316,47 @@ func (d *Downloader) byAttribute(c *fasthttp.RequestCtx, f func(request, pool.Ob return } - address, err := d.searchObject(c, log, containerID, key, val) + res, err := d.search(c, containerID, key, val, object.MatchStringEqual) if err != nil { - log.Error("couldn't search object", zap.Error(err)) - if errors.Is(err, errObjectNotFound) { - httpStatus = fasthttp.StatusNotFound - } - response.Error(c, "couldn't search object", httpStatus) + log.Error("could not search for objects", zap.Error(err)) + response.Error(c, "could not search for objects", fasthttp.StatusBadRequest) return } - f(*d.newRequest(c, log), d.pool, address) -} + defer res.Close() -func (d *Downloader) searchObject(c *fasthttp.RequestCtx, log *zap.Logger, cid *cid.ID, key, val string) (*object.Address, error) { - ids, err := d.searchByAttr(c, cid, key, val) - if err != nil { - return nil, err - } - if len(ids) > 1 { - log.Debug("found multiple objects", - zap.Strings("object_ids", objectIDs(ids).Slice()), - zap.Stringer("show_object_id", ids[0])) + buf := make([]oid.ID, 1) + + n, err := res.Read(buf) + if n == 0 { + if errors.Is(err, io.EOF) { + log.Error("object not found", zap.Error(err)) + response.Error(c, "object not found", fasthttp.StatusNotFound) + return + } + + log.Error("read object list failed", zap.Error(err)) + response.Error(c, "read object list failed", fasthttp.StatusBadRequest) + return } - return formAddress(cid, ids[0]), nil + var addrObj address.Address + addrObj.SetContainerID(containerID) + addrObj.SetObjectID(&buf[0]) + + f(*d.newRequest(c, log), d.pool, &addrObj) } -func formAddress(cid *cid.ID, oid *object.ID) *object.Address { - address := object.NewAddress() - address.SetContainerID(cid) - address.SetObjectID(oid) - return address -} +func (d *Downloader) search(c *fasthttp.RequestCtx, cid *cid.ID, key, val string, op object.SearchMatchType) (*pool.ResObjectSearch, error) { + filters := object.NewSearchFilters() + filters.AddRootFilter() + filters.AddFilter(key, val, op) -func (d *Downloader) search(c *fasthttp.RequestCtx, cid *cid.ID, key, val string, op object.SearchMatchType) ([]*object.ID, error) { - options := object.NewSearchFilters() - options.AddRootFilter() - options.AddFilter(key, val, op) - - sops := new(client.SearchObjectParams).WithContainerID(cid).WithSearchFilters(options) - ids, err := d.pool.SearchObject(c, sops) - if err != nil { - return nil, err - } - if len(ids) == 0 { - return nil, errObjectNotFound - } - return ids, nil -} - -func (d *Downloader) searchByPrefix(c *fasthttp.RequestCtx, cid *cid.ID, val string) ([]*object.ID, error) { - return d.search(c, cid, object.AttributeFileName, val, object.MatchCommonPrefix) -} - -func (d *Downloader) searchByAttr(c *fasthttp.RequestCtx, cid *cid.ID, key, val string) ([]*object.ID, error) { - return d.search(c, cid, key, val, object.MatchStringEqual) + return d.pool.SearchObjects(c, *cid, filters) } // DownloadZipped handles zip by prefix requests. func (d *Downloader) DownloadZipped(c *fasthttp.RequestCtx) { - status := fasthttp.StatusBadRequest scid, _ := c.UserValue("cid").(string) prefix, _ := url.QueryUnescape(c.UserValue("prefix").(string)) log := d.log.With(zap.String("cid", scid), zap.String("prefix", prefix)) @@ -393,7 +364,7 @@ func (d *Downloader) DownloadZipped(c *fasthttp.RequestCtx) { containerID := cid.New() if err := containerID.Parse(scid); err != nil { log.Error("wrong container id", zap.Error(err)) - response.Error(c, "wrong container id", status) + response.Error(c, "wrong container id", fasthttp.StatusBadRequest) return } @@ -403,71 +374,110 @@ func (d *Downloader) DownloadZipped(c *fasthttp.RequestCtx) { return } - ids, err := d.searchByPrefix(c, containerID, prefix) + resSearch, err := d.search(c, containerID, object.AttributeFileName, prefix, object.MatchCommonPrefix) if err != nil { - log.Error("couldn't find objects", zap.Error(err)) - if errors.Is(err, errObjectNotFound) { - status = fasthttp.StatusNotFound - } - response.Error(c, "couldn't find objects", status) + log.Error("could not search for objects", zap.Error(err)) + response.Error(c, "could not search for objects", fasthttp.StatusBadRequest) return } + defer resSearch.Close() + c.Response.Header.Set(fasthttp.HeaderContentType, "application/zip") c.Response.Header.Set(fasthttp.HeaderContentDisposition, "attachment; filename=\"archive.zip\"") c.Response.SetStatusCode(http.StatusOK) - if err = d.streamFiles(c, containerID, ids); err != nil { - log.Error("couldn't stream files", zap.Error(err)) - response.Error(c, "couldn't stream", fasthttp.StatusInternalServerError) - return - } -} - -func (d *Downloader) streamFiles(c *fasthttp.RequestCtx, cid *cid.ID, ids []*object.ID) error { zipWriter := zip.NewWriter(c) compression := zip.Store if d.settings.ZipCompression { compression = zip.Deflate } - for _, id := range ids { - var r io.Reader - readerInitCtx, initReader := context.WithCancel(c) - options := new(client.GetObjectParams). - WithAddress(formAddress(cid, id)). - WithPayloadReaderHandler(func(reader io.Reader) { - r = reader - initReader() + var ( + addr address.Address + resGet *pool.ResGetObject + w io.Writer + bufZip []byte + ) + + addr.SetContainerID(containerID) + + optBearer := bearerOpts(c) + empty := true + n := 0 + buf := make([]oid.ID, 10) // configure? + +iterator: + for { + n, err = resSearch.Read(buf) + if err != nil { + if errors.Is(err, io.EOF) { + if empty { + log.Error("objects not found", zap.Error(err)) + response.Error(c, "objects not found", fasthttp.StatusNotFound) + return + } + + err = nil + + break + } + + log.Error("read object list failed", zap.Error(err)) + response.Error(c, "read object list failed", fasthttp.StatusBadRequest) // maybe best effort? + return + } + + if empty { + bufZip = make([]byte, 1024) // configure? + } + + empty = false + + for i := range buf[:n] { + addr.SetObjectID(&buf[i]) + + resGet, err = d.pool.GetObject(c, addr, optBearer) + if err != nil { + err = fmt.Errorf("get NeoFS object: %v", err) + break iterator + } + + w, err = zipWriter.CreateHeader(&zip.FileHeader{ + Name: getFilename(&resGet.Header), + Method: compression, + Modified: time.Now(), }) + if err != nil { + err = fmt.Errorf("zip create header: %v", err) + break iterator + } - obj, err := d.pool.GetObject(c, options, bearerOpts(c)) - if err != nil { - return err - } + _, err = io.CopyBuffer(w, resGet.Payload, bufZip) + if err != nil { + err = fmt.Errorf("copy object payload to zip file: %v", err) + break iterator + } - header := &zip.FileHeader{ - Name: getFilename(obj), - Method: compression, - Modified: time.Now(), - } - entryWriter, err := zipWriter.CreateHeader(header) - if err != nil { - return err - } + _ = resGet.Payload.Close() - <-readerInitCtx.Done() - _, err = io.Copy(entryWriter, r) - if err != nil { - return err - } - - if err = zipWriter.Flush(); err != nil { - return err + err = zipWriter.Flush() + if err != nil { + err = fmt.Errorf("flush zip writer: %v", err) + break iterator + } } } - return zipWriter.Close() + if err == nil { + err = zipWriter.Close() + } + + if err != nil { + log.Error("file streaming failure", zap.Error(err)) + response.Error(c, "file streaming failure", fasthttp.StatusInternalServerError) + return + } } func getFilename(obj *object.Object) string { diff --git a/downloader/head.go b/downloader/head.go index 9093eb1..7d9fdae 100644 --- a/downloader/head.go +++ b/downloader/head.go @@ -1,6 +1,7 @@ package downloader import ( + "io" "net/http" "strconv" "time" @@ -8,8 +9,8 @@ import ( "github.com/nspcc-dev/neofs-http-gw/response" "github.com/nspcc-dev/neofs-http-gw/tokens" "github.com/nspcc-dev/neofs-http-gw/utils" - "github.com/nspcc-dev/neofs-sdk-go/client" "github.com/nspcc-dev/neofs-sdk-go/object" + "github.com/nspcc-dev/neofs-sdk-go/object/address" "github.com/nspcc-dev/neofs-sdk-go/pool" "github.com/valyala/fasthttp" "go.uber.org/zap" @@ -23,7 +24,7 @@ const ( hdrContainerID = "X-Container-Id" ) -func (r request) headObject(clnt pool.Object, objectAddress *object.Address) { +func (r request) headObject(clnt pool.Object, objectAddress *address.Address) { var start = time.Now() if err := tokens.StoreBearerToken(r.RequestCtx); err != nil { r.log.Error("could not fetch and store bearer token", zap.Error(err)) @@ -31,9 +32,8 @@ func (r request) headObject(clnt pool.Object, objectAddress *object.Address) { return } - options := new(client.ObjectHeaderParams).WithAddress(objectAddress) bearerOpt := bearerOpts(r.RequestCtx) - obj, err := clnt.GetObjectHeader(r.RequestCtx, options, bearerOpt) + obj, err := clnt.HeadObject(r.RequestCtx, *objectAddress, bearerOpt) if err != nil { r.handleNeoFSErr(err, start) return @@ -67,15 +67,22 @@ func (r request) headObject(clnt pool.Object, objectAddress *object.Address) { idsToResponse(&r.Response, obj) if len(contentType) == 0 { - objRange := object.NewRange() - objRange.SetOffset(0) - if sizeToDetectType < obj.PayloadSize() { - objRange.SetLength(sizeToDetectType) - } else { - objRange.SetLength(obj.PayloadSize()) + sz := obj.PayloadSize() + if sz > sizeToDetectType { + sz = sizeToDetectType } - ops := new(client.RangeDataParams).WithAddress(objectAddress).WithRange(objRange) - data, err := clnt.ObjectPayloadRangeData(r.RequestCtx, ops, bearerOpt) + + res, err := clnt.ObjectRange(r.RequestCtx, *objectAddress, 0, sz, bearerOpt) + if err != nil { + r.handleNeoFSErr(err, start) + return + } + + defer res.Close() + + data := make([]byte, sz) // sync-pool it? + + _, err = io.ReadFull(res, data) if err != nil { r.handleNeoFSErr(err, start) return diff --git a/go.mod b/go.mod index 256f6b9..33310c6 100644 --- a/go.mod +++ b/go.mod @@ -7,22 +7,28 @@ require ( github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect github.com/fasthttp/router v1.4.1 github.com/golang/mock v1.6.0 // indirect - github.com/golang/snappy v0.0.4 // indirect + github.com/gorilla/mux v1.8.0 // indirect + github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect github.com/klauspost/compress v1.13.1 // indirect github.com/nspcc-dev/neo-go v0.98.0 - github.com/nspcc-dev/neofs-api-go/v2 v2.11.0-pre.0.20211201134523-3604d96f3fe1 - github.com/nspcc-dev/neofs-sdk-go v0.0.0-20211201182451-a5b61c4f6477 + github.com/nspcc-dev/neofs-api-go/v2 v2.11.2-0.20220127135316-32dd0bb3f9c5 + github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.1.0.20220221122137-66bc59da5c02 github.com/prometheus/client_golang v1.11.0 github.com/prometheus/common v0.29.0 github.com/prometheus/procfs v0.7.1 // indirect github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.8.1 github.com/stretchr/testify v1.7.0 + github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect github.com/testcontainers/testcontainers-go v0.12.0 github.com/valyala/fasthttp v1.28.0 go.uber.org/multierr v1.7.0 // indirect go.uber.org/zap v1.18.1 + golang.org/x/crypto v0.0.0-20220209155544-dad33157f4bf // indirect + golang.org/x/sys v0.0.0-20220207234003-57398862261d // indirect golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b // indirect + golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect golang.org/x/tools v0.1.5 // indirect google.golang.org/grpc v1.41.0 + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect ) diff --git a/go.sum b/go.sum index 044e645..f90fd90 100644 --- a/go.sum +++ b/go.sum @@ -429,8 +429,9 @@ github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3i github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/handlers v0.0.0-20150720190736-60c7bfde3e33/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= -github.com/gorilla/mux v1.7.2 h1:zoNxOV7WjqXptQOVngLmcSQgXmgk4NMz1HibBchjl/I= github.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= @@ -455,8 +456,9 @@ github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d h1:dg1dEPuWpEqDnvIw251EVy4zlP8gWbsGj4BsUKCRpYs= +github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= @@ -586,14 +588,16 @@ github.com/nspcc-dev/hrw v1.0.9/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkP github.com/nspcc-dev/neo-go v0.73.1-pre.0.20200303142215-f5a1b928ce09/go.mod h1:pPYwPZ2ks+uMnlRLUyXOpLieaDQSEaf4NM3zHVbRjmg= github.com/nspcc-dev/neo-go v0.98.0 h1:yyW4sgY88/pLf0949qmgfkQXzRKC3CI/WyhqXNnwMd8= github.com/nspcc-dev/neo-go v0.98.0/go.mod h1:E3cc1x6RXSXrJb2nDWXTXjnXk3rIqVN8YdFyWv+FrqM= -github.com/nspcc-dev/neofs-api-go/v2 v2.11.0-pre.0.20211201134523-3604d96f3fe1 h1:CGA56mhLLduWRuMHcWujP5Ek+gAnXHk0WuIWkG65G1s= github.com/nspcc-dev/neofs-api-go/v2 v2.11.0-pre.0.20211201134523-3604d96f3fe1/go.mod h1:oS8dycEh8PPf2Jjp6+8dlwWyEv2Dy77h/XhhcdxYEFs= +github.com/nspcc-dev/neofs-api-go/v2 v2.11.2-0.20220127135316-32dd0bb3f9c5 h1:y9tbmUYhcr052QXsa4/IfUKAi2cx3TGDsEZUAow3P/Y= +github.com/nspcc-dev/neofs-api-go/v2 v2.11.2-0.20220127135316-32dd0bb3f9c5/go.mod h1:oS8dycEh8PPf2Jjp6+8dlwWyEv2Dy77h/XhhcdxYEFs= github.com/nspcc-dev/neofs-crypto v0.2.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9KAEAUQR7dMxZmNA= github.com/nspcc-dev/neofs-crypto v0.2.3/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= github.com/nspcc-dev/neofs-crypto v0.3.0 h1:zlr3pgoxuzrmGCxc5W8dGVfA9Rro8diFvVnBg0L4ifM= github.com/nspcc-dev/neofs-crypto v0.3.0/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= -github.com/nspcc-dev/neofs-sdk-go v0.0.0-20211201182451-a5b61c4f6477 h1:JC+jt4ARpMV/L3OqPHBKxAmbMabU7RYl/L4KgBz3yPs= github.com/nspcc-dev/neofs-sdk-go v0.0.0-20211201182451-a5b61c4f6477/go.mod h1:dfMtQWmBHYpl9Dez23TGtIUKiFvCIxUZq/CkSIhEpz4= +github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.1.0.20220221122137-66bc59da5c02 h1:g9tIrZU45dVFUSiY7Bb8m43rV/CJiIoPgQrxnbtKfKE= +github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.1.0.20220221122137-66bc59da5c02/go.mod h1:NeDPJaKJ6yCOWXRmfc3aRrhBPEOeAPD7q/6bp1UQCbs= github.com/nspcc-dev/rfc6979 v0.1.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYvOE= github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= @@ -763,8 +767,9 @@ github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/goleveldb v0.0.0-20180307113352-169b1b37be73/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0= -github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 h1:xQdMZ1WLrgkkvOZ/LDQxjVxMLdby7osSh4ZEVa5sIjs= github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= +github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= +github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I= github.com/testcontainers/testcontainers-go v0.12.0 h1:SK0NryGHIx7aifF6YqReORL18aGAA4bsDPtikDVCEyg= github.com/testcontainers/testcontainers-go v0.12.0/go.mod h1:SIndOQXZng0IW8iWU1Js0ynrfZ8xcxrTtDfF6rD2pxs= @@ -854,8 +859,9 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= -golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 h1:/UOmuWzQfxxo9UtlXMwuQU8CMgg1eZXqTRwkSQJWKOI= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220209155544-dad33157f4bf h1:gdgmgieTI2lLaGI2N+xEiaCMUgo2XFmAS0rlF8HZoso= +golang.org/x/crypto v0.0.0-20220209155544-dad33157f4bf/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -944,8 +950,9 @@ golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLd golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211108170745-6635138e15ea h1:FosBMXtOc8Tp9Hbo4ltl1WJSrTVewZU8MPnTPY2HdH8= golang.org/x/net v0.0.0-20211108170745-6635138e15ea/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1063,8 +1070,9 @@ golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211109184856-51b60fd695b3 h1:T6tyxxvHMj2L1R2kZg0uNMpS8ZhB9lRa9XRGTCSA65w= golang.org/x/sys v0.0.0-20211109184856-51b60fd695b3/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220207234003-57398862261d h1:Bm7BNOQt2Qv7ZqysjeLjgCBanX+88Z/OtdvsrEv1Djc= +golang.org/x/sys v0.0.0-20220207234003-57398862261d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210429154555-c04ba851c2a4/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b h1:9zKuko04nR4gjZ4+DNjHqRlAJqbJETHwiNKDqTfOjfE= @@ -1082,8 +1090,9 @@ golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e h1:EHBhcS0mlXEAVwNyO2dLfjToGsyY4j24pTs2ScHnX7s= golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE= +golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180318012157-96caea41033d/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1272,8 +1281,9 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20141024133853-64131543e789/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= diff --git a/integration_test.go b/integration_test.go index ffcd92f..4866fc7 100644 --- a/integration_test.go +++ b/integration_test.go @@ -16,10 +16,11 @@ import ( "time" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" - "github.com/nspcc-dev/neofs-sdk-go/client" "github.com/nspcc-dev/neofs-sdk-go/container" cid "github.com/nspcc-dev/neofs-sdk-go/container/id" "github.com/nspcc-dev/neofs-sdk-go/object" + "github.com/nspcc-dev/neofs-sdk-go/object/address" + oid "github.com/nspcc-dev/neofs-sdk-go/object/id" "github.com/nspcc-dev/neofs-sdk-go/policy" "github.com/nspcc-dev/neofs-sdk-go/pool" "github.com/spf13/viper" @@ -108,21 +109,25 @@ func simplePut(ctx context.Context, t *testing.T, clientPool pool.Pool, CID *cid err = CID.Parse(addr.CID) require.NoError(t, err) - oid := object.NewID() - err = oid.Parse(addr.OID) + id := oid.NewID() + err = id.Parse(addr.OID) require.NoError(t, err) - objectAddress := object.NewAddress() + objectAddress := address.NewAddress() objectAddress.SetContainerID(CID) - objectAddress.SetObjectID(oid) + objectAddress.SetObjectID(id) payload := bytes.NewBuffer(nil) - ops := new(client.GetObjectParams).WithAddress(objectAddress).WithPayloadWriter(payload) - obj, err := clientPool.GetObject(ctx, ops) + + res, err := clientPool.GetObject(ctx, *objectAddress) require.NoError(t, err) + + _, err = io.Copy(payload, res.Payload) + require.NoError(t, err) + require.Equal(t, content, payload.String()) - for _, attribute := range obj.Attributes() { + for _, attribute := range res.Header.Attributes() { require.Equal(t, attributes[attribute.Key()], attribute.Value()) } } @@ -133,9 +138,9 @@ func simpleGet(ctx context.Context, t *testing.T, clientPool pool.Pool, CID *cid "some-attr": "some-get-value", } - oid := putObject(ctx, t, clientPool, CID, content, attributes) + id := putObject(ctx, t, clientPool, CID, content, attributes) - resp, err := http.Get("http://localhost:8082/get/" + CID.String() + "/" + oid.String()) + resp, err := http.Get("http://localhost:8082/get/" + CID.String() + "/" + id.String()) require.NoError(t, err) defer func() { err = resp.Body.Close() @@ -156,11 +161,11 @@ func getByAttr(ctx context.Context, t *testing.T, clientPool pool.Pool, CID *cid content := "content of file" attributes := map[string]string{keyAttr: valAttr} - oid := putObject(ctx, t, clientPool, CID, content, attributes) + id := putObject(ctx, t, clientPool, CID, content, attributes) expectedAttr := map[string]string{ "X-Attribute-" + keyAttr: valAttr, - "x-object-id": oid.String(), + "x-object-id": id.String(), "x-container-id": CID.String(), } @@ -271,10 +276,9 @@ func getPool(ctx context.Context, t *testing.T, key *keys.PrivateKey) pool.Pool pb.AddNode("localhost:8080", 1, 1) opts := &pool.BuilderOptions{ - Key: &key.PrivateKey, - NodeConnectionTimeout: 5 * time.Second, - NodeRequestTimeout: 5 * time.Second, - SessionExpirationEpoch: math.MaxUint64, + Key: &key.PrivateKey, + NodeConnectionTimeout: 5 * time.Second, + NodeRequestTimeout: 5 * time.Second, } clientPool, err := pb.Build(ctx, opts) require.NoError(t, err) @@ -305,7 +309,7 @@ func createContainer(ctx context.Context, t *testing.T, clientPool pool.Pool) *c return CID } -func putObject(ctx context.Context, t *testing.T, clientPool pool.Pool, CID *cid.ID, content string, attributes map[string]string) *object.ID { +func putObject(ctx context.Context, t *testing.T, clientPool pool.Pool, CID *cid.ID, content string, attributes map[string]string) *oid.ID { rawObject := object.NewRaw() rawObject.SetContainerID(CID) rawObject.SetOwnerID(clientPool.OwnerID()) @@ -319,9 +323,8 @@ func putObject(ctx context.Context, t *testing.T, clientPool pool.Pool, CID *cid } rawObject.SetAttributes(attrs...) - ops := new(client.PutObjectParams).WithObject(rawObject.Object()).WithPayloadReader(bytes.NewBufferString(content)) - oid, err := clientPool.PutObject(ctx, ops) + id, err := clientPool.PutObject(ctx, *rawObject.Object(), bytes.NewBufferString(content)) require.NoError(t, err) - return oid + return id } diff --git a/uploader/upload.go b/uploader/upload.go index 1861516..f6ed91e 100644 --- a/uploader/upload.go +++ b/uploader/upload.go @@ -17,6 +17,8 @@ import ( cid "github.com/nspcc-dev/neofs-sdk-go/container/id" "github.com/nspcc-dev/neofs-sdk-go/netmap" "github.com/nspcc-dev/neofs-sdk-go/object" + "github.com/nspcc-dev/neofs-sdk-go/object/address" + oid "github.com/nspcc-dev/neofs-sdk-go/object/id" "github.com/nspcc-dev/neofs-sdk-go/owner" "github.com/nspcc-dev/neofs-sdk-go/pool" "github.com/nspcc-dev/neofs-sdk-go/token" @@ -53,9 +55,9 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { var ( err error file MultipartFile - obj *object.ID - addr = object.NewAddress() - cid = cid.New() + obj *oid.ID + addr = address.NewAddress() + idCnr = cid.New() scid, _ = c.UserValue("cid").(string) log = u.log.With(zap.String("cid", scid)) bodyStream = c.RequestBodyStream() @@ -66,7 +68,7 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { response.Error(c, "could not fetch bearer token", fasthttp.StatusBadRequest) return } - if err = cid.Parse(scid); err != nil { + if err = idCnr.Parse(scid); err != nil { log.Error("wrong container id", zap.Error(err)) response.Error(c, "wrong container id", fasthttp.StatusBadRequest) return @@ -127,23 +129,21 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) { timestamp.SetValue(strconv.FormatInt(time.Now().Unix(), 10)) attributes = append(attributes, timestamp) } - oid, bt := u.fetchOwnerAndBearerToken(c) + id, bt := u.fetchOwnerAndBearerToken(c) rawObject := object.NewRaw() - rawObject.SetContainerID(cid) - rawObject.SetOwnerID(oid) + rawObject.SetContainerID(idCnr) + rawObject.SetOwnerID(id) rawObject.SetAttributes(attributes...) - ops := new(client.PutObjectParams).WithObject(rawObject.Object()).WithPayloadReader(file) - - if obj, err = u.pool.PutObject(c, ops, pool.WithBearer(bt)); err != nil { + if obj, err = u.pool.PutObject(c, *rawObject.Object(), file, pool.WithBearer(bt)); err != nil { log.Error("could not store file in neofs", zap.Error(err)) response.Error(c, "could not store file in neofs", fasthttp.StatusBadRequest) return } addr.SetObjectID(obj) - addr.SetContainerID(cid) + addr.SetContainerID(idCnr) // Try to return the response, otherwise, if something went wrong, throw an error. if err = newPutResponse(addr).encode(c); err != nil { @@ -181,7 +181,7 @@ type putResponse struct { ContainerID string `json:"container_id"` } -func newPutResponse(addr *object.Address) *putResponse { +func newPutResponse(addr *address.Address) *putResponse { return &putResponse{ ObjectID: addr.ObjectID().String(), ContainerID: addr.ContainerID().String(), @@ -197,7 +197,7 @@ func (pr *putResponse) encode(w io.Writer) error { func getEpochDurations(ctx context.Context, p pool.Pool) (*epochDurations, error) { if conn, _, err := p.Connection(); err != nil { return nil, err - } else if networkInfoRes, err := conn.NetworkInfo(ctx); err != nil { + } else if networkInfoRes, err := conn.NetworkInfo(ctx, client.PrmNetworkInfo{}); err != nil { return nil, err } else if err = apistatus.ErrFromStatus(networkInfoRes.Status()); err != nil { return nil, err