[#126] Upgrade NeoFS SDK Go library

Core changes:
  - `object.ID` moved to new package `oid`;
  - `object.Address` moved to new package `address`;
  - `pool.Object` interface changes.

Signed-off-by: Leonard Lyubich <leonard@nspcc.ru>
This commit is contained in:
Leonard Lyubich 2022-02-08 19:24:23 +03:00 committed by LeL
parent 39c29c6d6a
commit 2b7e4a36fb
7 changed files with 220 additions and 186 deletions

2
app.go
View file

@ -4,7 +4,6 @@ import (
"context"
"crypto/ecdsa"
"fmt"
"math"
"strconv"
"github.com/fasthttp/router"
@ -122,7 +121,6 @@ func newApp(ctx context.Context, opt ...Option) App {
NodeConnectionTimeout: a.cfg.GetDuration(cfgConTimeout),
NodeRequestTimeout: a.cfg.GetDuration(cfgReqTimeout),
ClientRebalanceInterval: a.cfg.GetDuration(cfgRebalance),
SessionExpirationEpoch: math.MaxUint64,
}
a.pool, err = pb.Build(ctx, opts)
if err != nil {

View file

@ -16,9 +16,10 @@ import (
"github.com/nspcc-dev/neofs-http-gw/response"
"github.com/nspcc-dev/neofs-http-gw/tokens"
"github.com/nspcc-dev/neofs-http-gw/utils"
"github.com/nspcc-dev/neofs-sdk-go/client"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
"github.com/nspcc-dev/neofs-sdk-go/object"
"github.com/nspcc-dev/neofs-sdk-go/object/address"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"github.com/nspcc-dev/neofs-sdk-go/pool"
"github.com/valyala/fasthttp"
"go.uber.org/zap"
@ -38,8 +39,6 @@ type (
log *zap.Logger
}
objectIDs []*object.ID
errReader struct {
data []byte
err error
@ -116,39 +115,39 @@ func isValidValue(s string) bool {
return true
}
func (r request) receiveFile(clnt pool.Object, objectAddress *object.Address) {
func (r request) receiveFile(clnt pool.Object, objectAddress *address.Address) {
var (
err error
dis = "inline"
start = time.Now()
filename string
obj *object.Object
)
if err = tokens.StoreBearerToken(r.RequestCtx); err != nil {
r.log.Error("could not fetch and store bearer token", zap.Error(err))
response.Error(r.RequestCtx, "could not fetch and store bearer token", fasthttp.StatusBadRequest)
return
}
readDetector := newDetector()
options := new(client.GetObjectParams).
WithAddress(objectAddress).
WithPayloadReaderHandler(func(reader io.Reader) {
readDetector.SetReader(reader)
readDetector.Detect()
})
obj, err = clnt.GetObject(r.RequestCtx, options, bearerOpts(r.RequestCtx))
rObj, err := clnt.GetObject(r.RequestCtx, *objectAddress, bearerOpts(r.RequestCtx))
if err != nil {
r.handleNeoFSErr(err, start)
return
}
// we can't close reader in this function, so how to do it?
if r.Request.URI().QueryArgs().GetBool("download") {
dis = "attachment"
}
r.Response.SetBodyStream(readDetector.MultiReader(), int(obj.PayloadSize()))
r.Response.Header.Set(fasthttp.HeaderContentLength, strconv.FormatUint(obj.PayloadSize(), 10))
readDetector := newDetector()
readDetector.SetReader(rObj.Payload)
readDetector.Detect()
r.Response.SetBodyStream(readDetector.MultiReader(), int(rObj.Header.PayloadSize()))
r.Response.Header.Set(fasthttp.HeaderContentLength, strconv.FormatUint(rObj.Header.PayloadSize(), 10))
var contentType string
for _, attr := range obj.Attributes() {
for _, attr := range rObj.Header.Attributes() {
key := attr.Key()
val := attr.Value()
if !isValidToken(key) || !isValidValue(val) {
@ -177,7 +176,7 @@ func (r request) receiveFile(clnt pool.Object, objectAddress *object.Address) {
}
}
idsToResponse(&r.Response, obj)
idsToResponse(&r.Response, &rObj.Header)
if len(contentType) == 0 {
if readDetector.err != nil {
@ -244,14 +243,6 @@ func (r *request) handleNeoFSErr(err error, start time.Time) {
response.Error(r.RequestCtx, msg, code)
}
func (o objectIDs) Slice() []string {
res := make([]string, 0, len(o))
for _, oid := range o {
res = append(res, oid.String())
}
return res
}
// Downloader is a download request handler.
type Downloader struct {
log *zap.Logger
@ -287,21 +278,21 @@ func (d *Downloader) DownloadByAddress(c *fasthttp.RequestCtx) {
// byAddress is wrapper for function (e.g. request.headObject, request.receiveFile) that
// prepares request and object address to it.
func (d *Downloader) byAddress(c *fasthttp.RequestCtx, f func(request, pool.Object, *object.Address)) {
func (d *Downloader) byAddress(c *fasthttp.RequestCtx, f func(request, pool.Object, *address.Address)) {
var (
address = object.NewAddress()
cid, _ = c.UserValue("cid").(string)
oid, _ = c.UserValue("oid").(string)
val = strings.Join([]string{cid, oid}, "/")
log = d.log.With(zap.String("cid", cid), zap.String("oid", oid))
addr = address.NewAddress()
idCnr, _ = c.UserValue("cid").(string)
idObj, _ = c.UserValue("oid").(string)
val = strings.Join([]string{idCnr, idObj}, "/")
log = d.log.With(zap.String("cid", idCnr), zap.String("oid", idObj))
)
if err := address.Parse(val); err != nil {
if err := addr.Parse(val); err != nil {
log.Error("wrong object address", zap.Error(err))
response.Error(c, "wrong object address", fasthttp.StatusBadRequest)
return
}
f(*d.newRequest(c, log), d.pool, address)
f(*d.newRequest(c, log), d.pool, addr)
}
// DownloadByAttribute handles attribute-based download requests.
@ -310,7 +301,7 @@ func (d *Downloader) DownloadByAttribute(c *fasthttp.RequestCtx) {
}
// byAttribute is wrapper similar to byAddress.
func (d *Downloader) byAttribute(c *fasthttp.RequestCtx, f func(request, pool.Object, *object.Address)) {
func (d *Downloader) byAttribute(c *fasthttp.RequestCtx, f func(request, pool.Object, *address.Address)) {
var (
httpStatus = fasthttp.StatusBadRequest
scid, _ = c.UserValue("cid").(string)
@ -325,67 +316,47 @@ func (d *Downloader) byAttribute(c *fasthttp.RequestCtx, f func(request, pool.Ob
return
}
address, err := d.searchObject(c, log, containerID, key, val)
res, err := d.search(c, containerID, key, val, object.MatchStringEqual)
if err != nil {
log.Error("couldn't search object", zap.Error(err))
if errors.Is(err, errObjectNotFound) {
httpStatus = fasthttp.StatusNotFound
}
response.Error(c, "couldn't search object", httpStatus)
log.Error("could not search for objects", zap.Error(err))
response.Error(c, "could not search for objects", fasthttp.StatusBadRequest)
return
}
f(*d.newRequest(c, log), d.pool, address)
}
defer res.Close()
func (d *Downloader) searchObject(c *fasthttp.RequestCtx, log *zap.Logger, cid *cid.ID, key, val string) (*object.Address, error) {
ids, err := d.searchByAttr(c, cid, key, val)
if err != nil {
return nil, err
}
if len(ids) > 1 {
log.Debug("found multiple objects",
zap.Strings("object_ids", objectIDs(ids).Slice()),
zap.Stringer("show_object_id", ids[0]))
buf := make([]oid.ID, 1)
n, err := res.Read(buf)
if n == 0 {
if errors.Is(err, io.EOF) {
log.Error("object not found", zap.Error(err))
response.Error(c, "object not found", fasthttp.StatusNotFound)
return
}
return formAddress(cid, ids[0]), nil
}
func formAddress(cid *cid.ID, oid *object.ID) *object.Address {
address := object.NewAddress()
address.SetContainerID(cid)
address.SetObjectID(oid)
return address
}
func (d *Downloader) search(c *fasthttp.RequestCtx, cid *cid.ID, key, val string, op object.SearchMatchType) ([]*object.ID, error) {
options := object.NewSearchFilters()
options.AddRootFilter()
options.AddFilter(key, val, op)
sops := new(client.SearchObjectParams).WithContainerID(cid).WithSearchFilters(options)
ids, err := d.pool.SearchObject(c, sops)
if err != nil {
return nil, err
log.Error("read object list failed", zap.Error(err))
response.Error(c, "read object list failed", fasthttp.StatusBadRequest)
return
}
if len(ids) == 0 {
return nil, errObjectNotFound
}
return ids, nil
var addrObj address.Address
addrObj.SetContainerID(containerID)
addrObj.SetObjectID(&buf[0])
f(*d.newRequest(c, log), d.pool, &addrObj)
}
func (d *Downloader) searchByPrefix(c *fasthttp.RequestCtx, cid *cid.ID, val string) ([]*object.ID, error) {
return d.search(c, cid, object.AttributeFileName, val, object.MatchCommonPrefix)
}
func (d *Downloader) search(c *fasthttp.RequestCtx, cid *cid.ID, key, val string, op object.SearchMatchType) (*pool.ResObjectSearch, error) {
filters := object.NewSearchFilters()
filters.AddRootFilter()
filters.AddFilter(key, val, op)
func (d *Downloader) searchByAttr(c *fasthttp.RequestCtx, cid *cid.ID, key, val string) ([]*object.ID, error) {
return d.search(c, cid, key, val, object.MatchStringEqual)
return d.pool.SearchObjects(c, *cid, filters)
}
// DownloadZipped handles zip by prefix requests.
func (d *Downloader) DownloadZipped(c *fasthttp.RequestCtx) {
status := fasthttp.StatusBadRequest
scid, _ := c.UserValue("cid").(string)
prefix, _ := url.QueryUnescape(c.UserValue("prefix").(string))
log := d.log.With(zap.String("cid", scid), zap.String("prefix", prefix))
@ -393,7 +364,7 @@ func (d *Downloader) DownloadZipped(c *fasthttp.RequestCtx) {
containerID := cid.New()
if err := containerID.Parse(scid); err != nil {
log.Error("wrong container id", zap.Error(err))
response.Error(c, "wrong container id", status)
response.Error(c, "wrong container id", fasthttp.StatusBadRequest)
return
}
@ -403,71 +374,110 @@ func (d *Downloader) DownloadZipped(c *fasthttp.RequestCtx) {
return
}
ids, err := d.searchByPrefix(c, containerID, prefix)
resSearch, err := d.search(c, containerID, object.AttributeFileName, prefix, object.MatchCommonPrefix)
if err != nil {
log.Error("couldn't find objects", zap.Error(err))
if errors.Is(err, errObjectNotFound) {
status = fasthttp.StatusNotFound
}
response.Error(c, "couldn't find objects", status)
log.Error("could not search for objects", zap.Error(err))
response.Error(c, "could not search for objects", fasthttp.StatusBadRequest)
return
}
defer resSearch.Close()
c.Response.Header.Set(fasthttp.HeaderContentType, "application/zip")
c.Response.Header.Set(fasthttp.HeaderContentDisposition, "attachment; filename=\"archive.zip\"")
c.Response.SetStatusCode(http.StatusOK)
if err = d.streamFiles(c, containerID, ids); err != nil {
log.Error("couldn't stream files", zap.Error(err))
response.Error(c, "couldn't stream", fasthttp.StatusInternalServerError)
return
}
}
func (d *Downloader) streamFiles(c *fasthttp.RequestCtx, cid *cid.ID, ids []*object.ID) error {
zipWriter := zip.NewWriter(c)
compression := zip.Store
if d.settings.ZipCompression {
compression = zip.Deflate
}
for _, id := range ids {
var r io.Reader
readerInitCtx, initReader := context.WithCancel(c)
options := new(client.GetObjectParams).
WithAddress(formAddress(cid, id)).
WithPayloadReaderHandler(func(reader io.Reader) {
r = reader
initReader()
})
var (
addr address.Address
resGet *pool.ResGetObject
w io.Writer
bufZip []byte
)
obj, err := d.pool.GetObject(c, options, bearerOpts(c))
addr.SetContainerID(containerID)
optBearer := bearerOpts(c)
empty := true
n := 0
buf := make([]oid.ID, 10) // configure?
iterator:
for {
n, err = resSearch.Read(buf)
if err != nil {
return err
if errors.Is(err, io.EOF) {
if empty {
log.Error("objects not found", zap.Error(err))
response.Error(c, "objects not found", fasthttp.StatusNotFound)
return
}
header := &zip.FileHeader{
Name: getFilename(obj),
err = nil
break
}
log.Error("read object list failed", zap.Error(err))
response.Error(c, "read object list failed", fasthttp.StatusBadRequest) // maybe best effort?
return
}
if empty {
bufZip = make([]byte, 1024) // configure?
}
empty = false
for i := range buf[:n] {
addr.SetObjectID(&buf[i])
resGet, err = d.pool.GetObject(c, addr, optBearer)
if err != nil {
err = fmt.Errorf("get NeoFS object: %v", err)
break iterator
}
w, err = zipWriter.CreateHeader(&zip.FileHeader{
Name: getFilename(&resGet.Header),
Method: compression,
Modified: time.Now(),
}
entryWriter, err := zipWriter.CreateHeader(header)
})
if err != nil {
return err
err = fmt.Errorf("zip create header: %v", err)
break iterator
}
<-readerInitCtx.Done()
_, err = io.Copy(entryWriter, r)
_, err = io.CopyBuffer(w, resGet.Payload, bufZip)
if err != nil {
return err
err = fmt.Errorf("copy object payload to zip file: %v", err)
break iterator
}
if err = zipWriter.Flush(); err != nil {
return err
_ = resGet.Payload.Close()
err = zipWriter.Flush()
if err != nil {
err = fmt.Errorf("flush zip writer: %v", err)
break iterator
}
}
}
return zipWriter.Close()
if err == nil {
err = zipWriter.Close()
}
if err != nil {
log.Error("file streaming failure", zap.Error(err))
response.Error(c, "file streaming failure", fasthttp.StatusInternalServerError)
return
}
}
func getFilename(obj *object.Object) string {

View file

@ -1,6 +1,7 @@
package downloader
import (
"io"
"net/http"
"strconv"
"time"
@ -8,8 +9,8 @@ import (
"github.com/nspcc-dev/neofs-http-gw/response"
"github.com/nspcc-dev/neofs-http-gw/tokens"
"github.com/nspcc-dev/neofs-http-gw/utils"
"github.com/nspcc-dev/neofs-sdk-go/client"
"github.com/nspcc-dev/neofs-sdk-go/object"
"github.com/nspcc-dev/neofs-sdk-go/object/address"
"github.com/nspcc-dev/neofs-sdk-go/pool"
"github.com/valyala/fasthttp"
"go.uber.org/zap"
@ -23,7 +24,7 @@ const (
hdrContainerID = "X-Container-Id"
)
func (r request) headObject(clnt pool.Object, objectAddress *object.Address) {
func (r request) headObject(clnt pool.Object, objectAddress *address.Address) {
var start = time.Now()
if err := tokens.StoreBearerToken(r.RequestCtx); err != nil {
r.log.Error("could not fetch and store bearer token", zap.Error(err))
@ -31,9 +32,8 @@ func (r request) headObject(clnt pool.Object, objectAddress *object.Address) {
return
}
options := new(client.ObjectHeaderParams).WithAddress(objectAddress)
bearerOpt := bearerOpts(r.RequestCtx)
obj, err := clnt.GetObjectHeader(r.RequestCtx, options, bearerOpt)
obj, err := clnt.HeadObject(r.RequestCtx, *objectAddress, bearerOpt)
if err != nil {
r.handleNeoFSErr(err, start)
return
@ -67,15 +67,22 @@ func (r request) headObject(clnt pool.Object, objectAddress *object.Address) {
idsToResponse(&r.Response, obj)
if len(contentType) == 0 {
objRange := object.NewRange()
objRange.SetOffset(0)
if sizeToDetectType < obj.PayloadSize() {
objRange.SetLength(sizeToDetectType)
} else {
objRange.SetLength(obj.PayloadSize())
sz := obj.PayloadSize()
if sz > sizeToDetectType {
sz = sizeToDetectType
}
ops := new(client.RangeDataParams).WithAddress(objectAddress).WithRange(objRange)
data, err := clnt.ObjectPayloadRangeData(r.RequestCtx, ops, bearerOpt)
res, err := clnt.ObjectRange(r.RequestCtx, *objectAddress, 0, sz, bearerOpt)
if err != nil {
r.handleNeoFSErr(err, start)
return
}
defer res.Close()
data := make([]byte, sz) // sync-pool it?
_, err = io.ReadFull(res, data)
if err != nil {
r.handleNeoFSErr(err, start)
return

12
go.mod
View file

@ -7,22 +7,28 @@ require (
github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect
github.com/fasthttp/router v1.4.1
github.com/golang/mock v1.6.0 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/gorilla/mux v1.8.0 // indirect
github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect
github.com/klauspost/compress v1.13.1 // indirect
github.com/nspcc-dev/neo-go v0.98.0
github.com/nspcc-dev/neofs-api-go/v2 v2.11.0-pre.0.20211201134523-3604d96f3fe1
github.com/nspcc-dev/neofs-sdk-go v0.0.0-20211201182451-a5b61c4f6477
github.com/nspcc-dev/neofs-api-go/v2 v2.11.2-0.20220127135316-32dd0bb3f9c5
github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.1.0.20220221122137-66bc59da5c02
github.com/prometheus/client_golang v1.11.0
github.com/prometheus/common v0.29.0
github.com/prometheus/procfs v0.7.1 // indirect
github.com/spf13/pflag v1.0.5
github.com/spf13/viper v1.8.1
github.com/stretchr/testify v1.7.0
github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect
github.com/testcontainers/testcontainers-go v0.12.0
github.com/valyala/fasthttp v1.28.0
go.uber.org/multierr v1.7.0 // indirect
go.uber.org/zap v1.18.1
golang.org/x/crypto v0.0.0-20220209155544-dad33157f4bf // indirect
golang.org/x/sys v0.0.0-20220207234003-57398862261d // indirect
golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b // indirect
golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect
golang.org/x/tools v0.1.5 // indirect
google.golang.org/grpc v1.41.0
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
)

30
go.sum
View file

@ -429,8 +429,9 @@ github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3i
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/handlers v0.0.0-20150720190736-60c7bfde3e33/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ=
github.com/gorilla/mux v1.7.2 h1:zoNxOV7WjqXptQOVngLmcSQgXmgk4NMz1HibBchjl/I=
github.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
@ -455,8 +456,9 @@ github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b
github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d h1:dg1dEPuWpEqDnvIw251EVy4zlP8gWbsGj4BsUKCRpYs=
github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
@ -586,14 +588,16 @@ github.com/nspcc-dev/hrw v1.0.9/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkP
github.com/nspcc-dev/neo-go v0.73.1-pre.0.20200303142215-f5a1b928ce09/go.mod h1:pPYwPZ2ks+uMnlRLUyXOpLieaDQSEaf4NM3zHVbRjmg=
github.com/nspcc-dev/neo-go v0.98.0 h1:yyW4sgY88/pLf0949qmgfkQXzRKC3CI/WyhqXNnwMd8=
github.com/nspcc-dev/neo-go v0.98.0/go.mod h1:E3cc1x6RXSXrJb2nDWXTXjnXk3rIqVN8YdFyWv+FrqM=
github.com/nspcc-dev/neofs-api-go/v2 v2.11.0-pre.0.20211201134523-3604d96f3fe1 h1:CGA56mhLLduWRuMHcWujP5Ek+gAnXHk0WuIWkG65G1s=
github.com/nspcc-dev/neofs-api-go/v2 v2.11.0-pre.0.20211201134523-3604d96f3fe1/go.mod h1:oS8dycEh8PPf2Jjp6+8dlwWyEv2Dy77h/XhhcdxYEFs=
github.com/nspcc-dev/neofs-api-go/v2 v2.11.2-0.20220127135316-32dd0bb3f9c5 h1:y9tbmUYhcr052QXsa4/IfUKAi2cx3TGDsEZUAow3P/Y=
github.com/nspcc-dev/neofs-api-go/v2 v2.11.2-0.20220127135316-32dd0bb3f9c5/go.mod h1:oS8dycEh8PPf2Jjp6+8dlwWyEv2Dy77h/XhhcdxYEFs=
github.com/nspcc-dev/neofs-crypto v0.2.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9KAEAUQR7dMxZmNA=
github.com/nspcc-dev/neofs-crypto v0.2.3/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw=
github.com/nspcc-dev/neofs-crypto v0.3.0 h1:zlr3pgoxuzrmGCxc5W8dGVfA9Rro8diFvVnBg0L4ifM=
github.com/nspcc-dev/neofs-crypto v0.3.0/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw=
github.com/nspcc-dev/neofs-sdk-go v0.0.0-20211201182451-a5b61c4f6477 h1:JC+jt4ARpMV/L3OqPHBKxAmbMabU7RYl/L4KgBz3yPs=
github.com/nspcc-dev/neofs-sdk-go v0.0.0-20211201182451-a5b61c4f6477/go.mod h1:dfMtQWmBHYpl9Dez23TGtIUKiFvCIxUZq/CkSIhEpz4=
github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.1.0.20220221122137-66bc59da5c02 h1:g9tIrZU45dVFUSiY7Bb8m43rV/CJiIoPgQrxnbtKfKE=
github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.1.0.20220221122137-66bc59da5c02/go.mod h1:NeDPJaKJ6yCOWXRmfc3aRrhBPEOeAPD7q/6bp1UQCbs=
github.com/nspcc-dev/rfc6979 v0.1.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso=
github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYvOE=
github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso=
@ -763,8 +767,9 @@ github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG
github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
github.com/syndtr/goleveldb v0.0.0-20180307113352-169b1b37be73/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0=
github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 h1:xQdMZ1WLrgkkvOZ/LDQxjVxMLdby7osSh4ZEVa5sIjs=
github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM=
github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY=
github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc=
github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I=
github.com/testcontainers/testcontainers-go v0.12.0 h1:SK0NryGHIx7aifF6YqReORL18aGAA4bsDPtikDVCEyg=
github.com/testcontainers/testcontainers-go v0.12.0/go.mod h1:SIndOQXZng0IW8iWU1Js0ynrfZ8xcxrTtDfF6rD2pxs=
@ -854,8 +859,9 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 h1:/UOmuWzQfxxo9UtlXMwuQU8CMgg1eZXqTRwkSQJWKOI=
golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20220209155544-dad33157f4bf h1:gdgmgieTI2lLaGI2N+xEiaCMUgo2XFmAS0rlF8HZoso=
golang.org/x/crypto v0.0.0-20220209155544-dad33157f4bf/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@ -944,8 +950,9 @@ golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLd
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211108170745-6635138e15ea h1:FosBMXtOc8Tp9Hbo4ltl1WJSrTVewZU8MPnTPY2HdH8=
golang.org/x/net v0.0.0-20211108170745-6635138e15ea/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@ -1063,8 +1070,9 @@ golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211109184856-51b60fd695b3 h1:T6tyxxvHMj2L1R2kZg0uNMpS8ZhB9lRa9XRGTCSA65w=
golang.org/x/sys v0.0.0-20211109184856-51b60fd695b3/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220207234003-57398862261d h1:Bm7BNOQt2Qv7ZqysjeLjgCBanX+88Z/OtdvsrEv1Djc=
golang.org/x/sys v0.0.0-20220207234003-57398862261d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210429154555-c04ba851c2a4/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b h1:9zKuko04nR4gjZ4+DNjHqRlAJqbJETHwiNKDqTfOjfE=
@ -1082,8 +1090,9 @@ golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxb
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e h1:EHBhcS0mlXEAVwNyO2dLfjToGsyY4j24pTs2ScHnX7s=
golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE=
golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180318012157-96caea41033d/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@ -1272,8 +1281,9 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20141024133853-64131543e789/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=

View file

@ -16,10 +16,11 @@ import (
"time"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
"github.com/nspcc-dev/neofs-sdk-go/client"
"github.com/nspcc-dev/neofs-sdk-go/container"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
"github.com/nspcc-dev/neofs-sdk-go/object"
"github.com/nspcc-dev/neofs-sdk-go/object/address"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"github.com/nspcc-dev/neofs-sdk-go/policy"
"github.com/nspcc-dev/neofs-sdk-go/pool"
"github.com/spf13/viper"
@ -108,21 +109,25 @@ func simplePut(ctx context.Context, t *testing.T, clientPool pool.Pool, CID *cid
err = CID.Parse(addr.CID)
require.NoError(t, err)
oid := object.NewID()
err = oid.Parse(addr.OID)
id := oid.NewID()
err = id.Parse(addr.OID)
require.NoError(t, err)
objectAddress := object.NewAddress()
objectAddress := address.NewAddress()
objectAddress.SetContainerID(CID)
objectAddress.SetObjectID(oid)
objectAddress.SetObjectID(id)
payload := bytes.NewBuffer(nil)
ops := new(client.GetObjectParams).WithAddress(objectAddress).WithPayloadWriter(payload)
obj, err := clientPool.GetObject(ctx, ops)
res, err := clientPool.GetObject(ctx, *objectAddress)
require.NoError(t, err)
_, err = io.Copy(payload, res.Payload)
require.NoError(t, err)
require.Equal(t, content, payload.String())
for _, attribute := range obj.Attributes() {
for _, attribute := range res.Header.Attributes() {
require.Equal(t, attributes[attribute.Key()], attribute.Value())
}
}
@ -133,9 +138,9 @@ func simpleGet(ctx context.Context, t *testing.T, clientPool pool.Pool, CID *cid
"some-attr": "some-get-value",
}
oid := putObject(ctx, t, clientPool, CID, content, attributes)
id := putObject(ctx, t, clientPool, CID, content, attributes)
resp, err := http.Get("http://localhost:8082/get/" + CID.String() + "/" + oid.String())
resp, err := http.Get("http://localhost:8082/get/" + CID.String() + "/" + id.String())
require.NoError(t, err)
defer func() {
err = resp.Body.Close()
@ -156,11 +161,11 @@ func getByAttr(ctx context.Context, t *testing.T, clientPool pool.Pool, CID *cid
content := "content of file"
attributes := map[string]string{keyAttr: valAttr}
oid := putObject(ctx, t, clientPool, CID, content, attributes)
id := putObject(ctx, t, clientPool, CID, content, attributes)
expectedAttr := map[string]string{
"X-Attribute-" + keyAttr: valAttr,
"x-object-id": oid.String(),
"x-object-id": id.String(),
"x-container-id": CID.String(),
}
@ -274,7 +279,6 @@ func getPool(ctx context.Context, t *testing.T, key *keys.PrivateKey) pool.Pool
Key: &key.PrivateKey,
NodeConnectionTimeout: 5 * time.Second,
NodeRequestTimeout: 5 * time.Second,
SessionExpirationEpoch: math.MaxUint64,
}
clientPool, err := pb.Build(ctx, opts)
require.NoError(t, err)
@ -305,7 +309,7 @@ func createContainer(ctx context.Context, t *testing.T, clientPool pool.Pool) *c
return CID
}
func putObject(ctx context.Context, t *testing.T, clientPool pool.Pool, CID *cid.ID, content string, attributes map[string]string) *object.ID {
func putObject(ctx context.Context, t *testing.T, clientPool pool.Pool, CID *cid.ID, content string, attributes map[string]string) *oid.ID {
rawObject := object.NewRaw()
rawObject.SetContainerID(CID)
rawObject.SetOwnerID(clientPool.OwnerID())
@ -319,9 +323,8 @@ func putObject(ctx context.Context, t *testing.T, clientPool pool.Pool, CID *cid
}
rawObject.SetAttributes(attrs...)
ops := new(client.PutObjectParams).WithObject(rawObject.Object()).WithPayloadReader(bytes.NewBufferString(content))
oid, err := clientPool.PutObject(ctx, ops)
id, err := clientPool.PutObject(ctx, *rawObject.Object(), bytes.NewBufferString(content))
require.NoError(t, err)
return oid
return id
}

View file

@ -17,6 +17,8 @@ import (
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
"github.com/nspcc-dev/neofs-sdk-go/netmap"
"github.com/nspcc-dev/neofs-sdk-go/object"
"github.com/nspcc-dev/neofs-sdk-go/object/address"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"github.com/nspcc-dev/neofs-sdk-go/owner"
"github.com/nspcc-dev/neofs-sdk-go/pool"
"github.com/nspcc-dev/neofs-sdk-go/token"
@ -53,9 +55,9 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) {
var (
err error
file MultipartFile
obj *object.ID
addr = object.NewAddress()
cid = cid.New()
obj *oid.ID
addr = address.NewAddress()
idCnr = cid.New()
scid, _ = c.UserValue("cid").(string)
log = u.log.With(zap.String("cid", scid))
bodyStream = c.RequestBodyStream()
@ -66,7 +68,7 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) {
response.Error(c, "could not fetch bearer token", fasthttp.StatusBadRequest)
return
}
if err = cid.Parse(scid); err != nil {
if err = idCnr.Parse(scid); err != nil {
log.Error("wrong container id", zap.Error(err))
response.Error(c, "wrong container id", fasthttp.StatusBadRequest)
return
@ -127,23 +129,21 @@ func (u *Uploader) Upload(c *fasthttp.RequestCtx) {
timestamp.SetValue(strconv.FormatInt(time.Now().Unix(), 10))
attributes = append(attributes, timestamp)
}
oid, bt := u.fetchOwnerAndBearerToken(c)
id, bt := u.fetchOwnerAndBearerToken(c)
rawObject := object.NewRaw()
rawObject.SetContainerID(cid)
rawObject.SetOwnerID(oid)
rawObject.SetContainerID(idCnr)
rawObject.SetOwnerID(id)
rawObject.SetAttributes(attributes...)
ops := new(client.PutObjectParams).WithObject(rawObject.Object()).WithPayloadReader(file)
if obj, err = u.pool.PutObject(c, ops, pool.WithBearer(bt)); err != nil {
if obj, err = u.pool.PutObject(c, *rawObject.Object(), file, pool.WithBearer(bt)); err != nil {
log.Error("could not store file in neofs", zap.Error(err))
response.Error(c, "could not store file in neofs", fasthttp.StatusBadRequest)
return
}
addr.SetObjectID(obj)
addr.SetContainerID(cid)
addr.SetContainerID(idCnr)
// Try to return the response, otherwise, if something went wrong, throw an error.
if err = newPutResponse(addr).encode(c); err != nil {
@ -181,7 +181,7 @@ type putResponse struct {
ContainerID string `json:"container_id"`
}
func newPutResponse(addr *object.Address) *putResponse {
func newPutResponse(addr *address.Address) *putResponse {
return &putResponse{
ObjectID: addr.ObjectID().String(),
ContainerID: addr.ContainerID().String(),
@ -197,7 +197,7 @@ func (pr *putResponse) encode(w io.Writer) error {
func getEpochDurations(ctx context.Context, p pool.Pool) (*epochDurations, error) {
if conn, _, err := p.Connection(); err != nil {
return nil, err
} else if networkInfoRes, err := conn.NetworkInfo(ctx); err != nil {
} else if networkInfoRes, err := conn.NetworkInfo(ctx, client.PrmNetworkInfo{}); err != nil {
return nil, err
} else if err = apistatus.ErrFromStatus(networkInfoRes.Status()); err != nil {
return nil, err