From 1ca3e6e04b249b3fcb3b62fbb94fdb6e8de12327 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Mon, 27 Jul 2020 09:49:23 +0300 Subject: [PATCH 1/5] NFSSVC-27 Simplify ListBuckets handler --- neofs/api/handler/container.go | 83 ++++++++++++++++------------------ 1 file changed, 38 insertions(+), 45 deletions(-) diff --git a/neofs/api/handler/container.go b/neofs/api/handler/container.go index 3f3fe04..f87cc15 100644 --- a/neofs/api/handler/container.go +++ b/neofs/api/handler/container.go @@ -43,7 +43,6 @@ type ( cnrInfoParams struct { cid refs.CID - con *grpc.ClientConn tkn *service.BearerTokenMsg } ) @@ -51,6 +50,7 @@ type ( func (h *handler) getContainerInfo(ctx context.Context, p cnrInfoParams) (*Bucket, error) { var ( err error + con *grpc.ClientConn res *container.GetResponse ) @@ -59,9 +59,11 @@ func (h *handler) getContainerInfo(ctx context.Context, p cnrInfoParams) (*Bucke req.SetTTL(service.SingleForwardingTTL) req.SetBearer(p.tkn) - if err = service.SignRequestData(h.key, req); err != nil { + if con, err = h.cli.GetConnection(ctx); err != nil { + return nil, errors.Wrap(err, "could not fetch connection") + } else if err = service.SignRequestData(h.key, req); err != nil { return nil, errors.Wrap(err, "could not sign container info request") - } else if res, err = container.NewServiceClient(p.con).Get(ctx, req); err != nil { + } else if res, err = container.NewServiceClient(con).Get(ctx, req); err != nil { return nil, errors.Wrap(err, "could not fetch container info") } @@ -75,13 +77,34 @@ func (h *handler) getContainerInfo(ctx context.Context, p cnrInfoParams) (*Bucke }, nil } +func (h *handler) getContainerList(ctx context.Context, tkn *service.BearerTokenMsg) ([]refs.CID, error) { + var ( + err error + con *grpc.ClientConn + res *container.ListResponse + ) + + req := new(container.ListRequest) + req.OwnerID = tkn.OwnerID + req.SetTTL(service.SingleForwardingTTL) + req.SetBearer(tkn) + + if con, err = h.cli.GetConnection(ctx); err != nil { + return nil, errors.Wrap(err, "could not fetch connection") + } else if err = service.SignRequestData(h.key, req); err != nil { + return nil, errors.Wrap(err, "could not sign request") + } else if res, err = container.NewServiceClient(con).List(ctx, req); err != nil { + return nil, errors.Wrap(err, "could not fetch list containers") + } + + return res.CID, nil +} + func (h *handler) ListBucketsHandler(w http.ResponseWriter, r *http.Request) { var ( err error - uid = h.uid inf *Bucket - con *grpc.ClientConn - res *container.ListResponse + lst []refs.CID tkn *service.BearerTokenMsg ) @@ -102,43 +125,11 @@ func (h *handler) ListBucketsHandler(w http.ResponseWriter, r *http.Request) { }, r.URL) return - } - - req := new(container.ListRequest) - req.OwnerID = uid - req.SetTTL(service.SingleForwardingTTL) - req.SetBearer(tkn) - // req.SetVersion(APIVersion) ?? - - if con, err = h.cli.GetConnection(ctx); err != nil { - h.log.Error("could not get connection", + } else if lst, err = h.getContainerList(ctx, tkn); err != nil { + h.log.Error("could not fetch bearer token", zap.Error(err)) - e := api.GetAPIError(api.ErrInternalError) - - api.WriteErrorResponse(ctx, w, api.Error{ - Code: e.Code, - Description: err.Error(), - HTTPStatusCode: e.HTTPStatusCode, - }, r.URL) - - return - } else if err = service.SignRequestData(h.key, req); err != nil { - h.log.Error("could not prepare request", - zap.Error(err)) - - e := api.GetAPIError(api.ErrInternalError) - - api.WriteErrorResponse(ctx, w, api.Error{ - Code: e.Code, - Description: err.Error(), - HTTPStatusCode: e.HTTPStatusCode, - }, r.URL) - - return - } else if res, err = container.NewServiceClient(con).List(ctx, req); err != nil { - h.log.Error("could not list buckets", - zap.Error(err)) + // TODO check that error isn't gRPC error e := api.GetAPIError(api.ErrInternalError) @@ -152,13 +143,13 @@ func (h *handler) ListBucketsHandler(w http.ResponseWriter, r *http.Request) { } result := &ListBucketsResponse{Owner: Owner{ - ID: uid.String(), - DisplayName: uid.String(), + ID: tkn.OwnerID.String(), + DisplayName: tkn.OwnerID.String(), }} - params := cnrInfoParams{con: con, tkn: tkn} + params := cnrInfoParams{tkn: tkn} - for _, cid := range res.CID { + for _, cid := range lst { // should receive each container info (??): params.cid = cid @@ -166,6 +157,8 @@ func (h *handler) ListBucketsHandler(w http.ResponseWriter, r *http.Request) { h.log.Error("could not fetch bucket info", zap.Error(err)) + // TODO check that error isn't gRPC error + e := api.GetAPIError(api.ErrInternalError) api.WriteErrorResponse(ctx, w, api.Error{ From 58fdcd090f1f21beeb3ac5977bfc9ca7aaff5579 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Tue, 28 Jul 2020 01:54:47 +0300 Subject: [PATCH 2/5] NFSSVC-27 Fixes for auth middleware --- neofs/api/user-auth.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neofs/api/user-auth.go b/neofs/api/user-auth.go index 99dd0fb..17d6eae 100644 --- a/neofs/api/user-auth.go +++ b/neofs/api/user-auth.go @@ -14,7 +14,7 @@ func AttachUserAuth(router *mux.Router, center *auth.Center, log *zap.Logger) { bearerToken, err := center.AuthenticationPassed(r) if err != nil { log.Error("failed to pass authentication", zap.Error(err)) - WriteErrorResponse(r.Context(), w, getAPIError(ErrAccessDenied), r.URL) + WriteErrorResponse(r.Context(), w, GetAPIError(ErrAccessDenied), r.URL) return } h.ServeHTTP(w, r.WithContext(auth.SetBearerToken(r.Context(), bearerToken))) From b12b5008e3b4ff6a251dce16ef40f036acbb5ae9 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Tue, 28 Jul 2020 01:55:08 +0300 Subject: [PATCH 3/5] NFSSVC-27 Update dependencies --- go.mod | 1 - go.sum | 6 ------ 2 files changed, 7 deletions(-) diff --git a/go.mod b/go.mod index 9405d03..6edcbe5 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,6 @@ require ( github.com/Shopify/sarama v1.24.1 github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect github.com/alecthomas/participle v0.2.1 - github.com/aws/aws-sdk-go v1.33.8 github.com/bcicen/jstream v0.0.0-20190220045926-16c1f8af81c2 github.com/beevik/ntp v0.2.0 github.com/cespare/xxhash/v2 v2.1.1 diff --git a/go.sum b/go.sum index 8ee0365..0c76b48 100644 --- a/go.sum +++ b/go.sum @@ -47,8 +47,6 @@ github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQh github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/awalterschulze/gographviz v0.0.0-20181013152038-b2885df04310 h1:t+qxRrRtwNiUYA+Xh2jSXhoG2grnMCMKX4Fg6lx9X1U= github.com/awalterschulze/gographviz v0.0.0-20181013152038-b2885df04310/go.mod h1:GEV5wmg4YquNw7v1kkyoX9etIk8yVmXj+AkDHuuETHs= -github.com/aws/aws-sdk-go v1.33.8 h1:2/sOfb9oPHTRZ0lxinoaTPDcYwNa1H/SpKP4nVRBwmg= -github.com/aws/aws-sdk-go v1.33.8/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/bcicen/jstream v0.0.0-20190220045926-16c1f8af81c2 h1:M+TYzBcNIRyzPRg66ndEqUMd7oWDmhvdQmaPC6EZNwM= github.com/bcicen/jstream v0.0.0-20190220045926-16c1f8af81c2/go.mod h1:RDu/qcrnpEdJC/p8tx34+YBFqqX71lB7dOX9QE+ZC4M= github.com/beevik/ntp v0.2.0 h1:sGsd+kAXzT0bfVfzJfce04g+dSRfrs+tbQW8lweuYgw= @@ -261,8 +259,6 @@ github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf h1:WfD7V github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf/go.mod h1:hyb9oH7vZsitZCiBt0ZvifOrB+qc8PS5IiilCIb87rg= github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03 h1:FUwcHNlEqkqLjLBdCp5PRlCFijNjvcYANOZXzCfXwCM= github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= -github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= -github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -401,7 +397,6 @@ github.com/nspcc-dev/hrw v1.0.8 h1:vwRuJXZXgkMvf473vFzeWGCfY1WBVeSHAEHvR4u3/Cg= github.com/nspcc-dev/hrw v1.0.8/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkPG06MU= github.com/nspcc-dev/neofs-api-go v1.2.0 h1:8vovd8hvnoWS4qkSa6rhyMFLFvjLtNKar5vYRodf+y4= github.com/nspcc-dev/neofs-api-go v1.2.0/go.mod h1:2tf31g2Ns/Z2ev5d8LZ/9f1VHIeY5LHpDbq4EsDhYM0= -github.com/nspcc-dev/neofs-api-go v1.3.0 h1:w0wYIXzPJIElwhqahnQw/1NKiHxjRZKJhDUMSbEHmdk= github.com/nspcc-dev/neofs-crypto v0.3.0 h1:zlr3pgoxuzrmGCxc5W8dGVfA9Rro8diFvVnBg0L4ifM= github.com/nspcc-dev/neofs-crypto v0.3.0/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= github.com/nspcc-dev/netmap v1.7.0 h1:ak64xn/gPdgYw4tsqSSF7kAGQGbEpeuJEF3XwBX4L9Y= @@ -604,7 +599,6 @@ golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200602114024-627f9648deb9 h1:pNX+40auqi2JqRfOP1akLGtYcn15TUbkhwuCO3foqqM= golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= From 504ef4da918318e1604a7c8be1b1338d6e282975 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Tue, 28 Jul 2020 01:56:10 +0300 Subject: [PATCH 4/5] NFSSVC-27 Return list of buckets from getContainerList --- neofs/api/handler/container.go | 46 +++++++++++++--------------------- 1 file changed, 17 insertions(+), 29 deletions(-) diff --git a/neofs/api/handler/container.go b/neofs/api/handler/container.go index f87cc15..f31b555 100644 --- a/neofs/api/handler/container.go +++ b/neofs/api/handler/container.go @@ -77,9 +77,10 @@ func (h *handler) getContainerInfo(ctx context.Context, p cnrInfoParams) (*Bucke }, nil } -func (h *handler) getContainerList(ctx context.Context, tkn *service.BearerTokenMsg) ([]refs.CID, error) { +func (h *handler) getContainerList(ctx context.Context, tkn *service.BearerTokenMsg) ([]*Bucket, error) { var ( err error + inf *Bucket con *grpc.ClientConn res *container.ListResponse ) @@ -97,14 +98,25 @@ func (h *handler) getContainerList(ctx context.Context, tkn *service.BearerToken return nil, errors.Wrap(err, "could not fetch list containers") } - return res.CID, nil + params := cnrInfoParams{tkn: tkn} + result := make([]*Bucket, 0, len(res.CID)) + + for _, cid := range res.CID { + params.cid = cid + if inf, err = h.getContainerInfo(ctx, params); err != nil { + return nil, errors.Wrap(err, "could not fetch container info") + } + + result = append(result, inf) + } + + return result, nil } func (h *handler) ListBucketsHandler(w http.ResponseWriter, r *http.Request) { var ( err error - inf *Bucket - lst []refs.CID + lst []*Bucket tkn *service.BearerTokenMsg ) @@ -147,31 +159,7 @@ func (h *handler) ListBucketsHandler(w http.ResponseWriter, r *http.Request) { DisplayName: tkn.OwnerID.String(), }} - params := cnrInfoParams{tkn: tkn} - - for _, cid := range lst { - // should receive each container info (??): - params.cid = cid - - if inf, err = h.getContainerInfo(ctx, params); err != nil { - h.log.Error("could not fetch bucket info", - zap.Error(err)) - - // TODO check that error isn't gRPC error - - e := api.GetAPIError(api.ErrInternalError) - - api.WriteErrorResponse(ctx, w, api.Error{ - Code: e.Code, - Description: err.Error(), - HTTPStatusCode: e.HTTPStatusCode, - }, r.URL) - - return - } - - result.Buckets.Buckets = append(result.Buckets.Buckets, inf) - } + result.Buckets.Buckets = lst // Generate response. encodedSuccessResponse := api.EncodeResponse(result) From 1db70efd7b1cb6c5a529c57fc08a648847f6ede4 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Tue, 28 Jul 2020 11:43:42 +0300 Subject: [PATCH 5/5] NFSSVC-27 Add CID to Bucket struct This is done in order to cache the list of containers in the future --- neofs/api/handler/container.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/neofs/api/handler/container.go b/neofs/api/handler/container.go index f31b555..2c8041b 100644 --- a/neofs/api/handler/container.go +++ b/neofs/api/handler/container.go @@ -25,6 +25,7 @@ type ( // Bucket container for bucket metadata Bucket struct { + CID refs.CID `xml:"-"` // ignored by response Name string CreationDate string // time string of format "2006-01-02T15:04:05.000Z" } @@ -72,6 +73,7 @@ func (h *handler) getContainerInfo(ctx context.Context, p cnrInfoParams) (*Bucke _ = res return &Bucket{ + CID: p.cid, Name: p.cid.String(), CreationDate: new(time.Time).Format(time.RFC3339), }, nil