forked from TrueCloudLab/frostfs-http-gw
[#142] Support resolving container nicename
Signed-off-by: Denis Kirillov <denis@nspcc.ru>
This commit is contained in:
parent
2b780c1772
commit
a42606742a
14 changed files with 458 additions and 74 deletions
|
@ -189,6 +189,8 @@ and upload objects with it, but deleting, searching, managing ACLs, creating
|
||||||
containers and other activities are not supported and not planned to be
|
containers and other activities are not supported and not planned to be
|
||||||
supported.
|
supported.
|
||||||
|
|
||||||
|
**Note:** in all download/upload routes you can use container name instead of it's id (`$CID`), but resolvers must be configured properly (see [configs](./config) for examples).
|
||||||
|
|
||||||
### Preparation
|
### Preparation
|
||||||
|
|
||||||
Before uploading or downloading a file make sure you have a prepared container.
|
Before uploading or downloading a file make sure you have a prepared container.
|
||||||
|
|
58
app.go
58
app.go
|
@ -13,8 +13,10 @@ import (
|
||||||
"github.com/nspcc-dev/neo-go/pkg/util"
|
"github.com/nspcc-dev/neo-go/pkg/util"
|
||||||
"github.com/nspcc-dev/neo-go/pkg/wallet"
|
"github.com/nspcc-dev/neo-go/pkg/wallet"
|
||||||
"github.com/nspcc-dev/neofs-http-gw/downloader"
|
"github.com/nspcc-dev/neofs-http-gw/downloader"
|
||||||
|
"github.com/nspcc-dev/neofs-http-gw/resolver"
|
||||||
"github.com/nspcc-dev/neofs-http-gw/response"
|
"github.com/nspcc-dev/neofs-http-gw/response"
|
||||||
"github.com/nspcc-dev/neofs-http-gw/uploader"
|
"github.com/nspcc-dev/neofs-http-gw/uploader"
|
||||||
|
"github.com/nspcc-dev/neofs-http-gw/utils"
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/pool"
|
"github.com/nspcc-dev/neofs-sdk-go/pool"
|
||||||
"github.com/spf13/viper"
|
"github.com/spf13/viper"
|
||||||
"github.com/valyala/fasthttp"
|
"github.com/valyala/fasthttp"
|
||||||
|
@ -28,6 +30,7 @@ type (
|
||||||
cfg *viper.Viper
|
cfg *viper.Viper
|
||||||
webServer *fasthttp.Server
|
webServer *fasthttp.Server
|
||||||
webDone chan struct{}
|
webDone chan struct{}
|
||||||
|
resolver *resolver.ContainerResolver
|
||||||
}
|
}
|
||||||
|
|
||||||
// App is an interface for the main gateway function.
|
// App is an interface for the main gateway function.
|
||||||
|
@ -127,9 +130,39 @@ func newApp(ctx context.Context, opt ...Option) App {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
a.log.Fatal("failed to dial pool", zap.Error(err))
|
a.log.Fatal("failed to dial pool", zap.Error(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
resolveCfg := &resolver.Config{
|
||||||
|
NeoFS: resolver.NewNeoFSResolver(a.pool),
|
||||||
|
RPCAddress: a.cfg.GetString(cfgRPCEndpoint),
|
||||||
|
}
|
||||||
|
|
||||||
|
order := a.cfg.GetStringSlice(cfgResolveOrder)
|
||||||
|
if resolveCfg.RPCAddress == "" {
|
||||||
|
order = remove(order, resolver.NNSResolver)
|
||||||
|
a.log.Warn(fmt.Sprintf("resolver '%s' won't be used since '%s' isn't provided", resolver.NNSResolver, cfgRPCEndpoint))
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(order) != 0 {
|
||||||
|
a.resolver, err = resolver.NewResolver(order, resolveCfg)
|
||||||
|
if err != nil {
|
||||||
|
a.log.Fatal("failed to create resolver", zap.Error(err))
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
a.log.Info("container resolver is disabled")
|
||||||
|
}
|
||||||
|
|
||||||
return a
|
return a
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func remove(list []string, element string) []string {
|
||||||
|
for i, item := range list {
|
||||||
|
if item == element {
|
||||||
|
return append(list[:i], list[i+1:]...)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return list
|
||||||
|
}
|
||||||
|
|
||||||
func getNeoFSKey(a *app) (*ecdsa.PrivateKey, error) {
|
func getNeoFSKey(a *app) (*ecdsa.PrivateKey, error) {
|
||||||
walletPath := a.cfg.GetString(cmdWallet)
|
walletPath := a.cfg.GetString(cmdWallet)
|
||||||
if len(walletPath) == 0 {
|
if len(walletPath) == 0 {
|
||||||
|
@ -208,8 +241,9 @@ func (a *app) Serve(ctx context.Context) {
|
||||||
close(a.webDone)
|
close(a.webDone)
|
||||||
}()
|
}()
|
||||||
edts := a.cfg.GetBool(cfgUploaderHeaderEnableDefaultTimestamp)
|
edts := a.cfg.GetBool(cfgUploaderHeaderEnableDefaultTimestamp)
|
||||||
uploader := uploader.New(ctx, a.log, a.pool, edts)
|
uploadRoutes := uploader.New(ctx, a.AppParams(), edts)
|
||||||
downloader := downloader.New(ctx, a.log, downloader.Settings{ZipCompression: a.cfg.GetBool(cfgZipCompression)}, a.pool)
|
downloadSettings := downloader.Settings{ZipCompression: a.cfg.GetBool(cfgZipCompression)}
|
||||||
|
downloadRoutes := downloader.New(ctx, a.AppParams(), downloadSettings)
|
||||||
// Configure router.
|
// Configure router.
|
||||||
r := router.New()
|
r := router.New()
|
||||||
r.RedirectTrailingSlash = true
|
r.RedirectTrailingSlash = true
|
||||||
|
@ -219,15 +253,15 @@ func (a *app) Serve(ctx context.Context) {
|
||||||
r.MethodNotAllowed = func(r *fasthttp.RequestCtx) {
|
r.MethodNotAllowed = func(r *fasthttp.RequestCtx) {
|
||||||
response.Error(r, "Method Not Allowed", fasthttp.StatusMethodNotAllowed)
|
response.Error(r, "Method Not Allowed", fasthttp.StatusMethodNotAllowed)
|
||||||
}
|
}
|
||||||
r.POST("/upload/{cid}", a.logger(uploader.Upload))
|
r.POST("/upload/{cid}", a.logger(uploadRoutes.Upload))
|
||||||
a.log.Info("added path /upload/{cid}")
|
a.log.Info("added path /upload/{cid}")
|
||||||
r.GET("/get/{cid}/{oid}", a.logger(downloader.DownloadByAddress))
|
r.GET("/get/{cid}/{oid}", a.logger(downloadRoutes.DownloadByAddress))
|
||||||
r.HEAD("/get/{cid}/{oid}", a.logger(downloader.HeadByAddress))
|
r.HEAD("/get/{cid}/{oid}", a.logger(downloadRoutes.HeadByAddress))
|
||||||
a.log.Info("added path /get/{cid}/{oid}")
|
a.log.Info("added path /get/{cid}/{oid}")
|
||||||
r.GET("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.logger(downloader.DownloadByAttribute))
|
r.GET("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.logger(downloadRoutes.DownloadByAttribute))
|
||||||
r.HEAD("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.logger(downloader.HeadByAttribute))
|
r.HEAD("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.logger(downloadRoutes.HeadByAttribute))
|
||||||
a.log.Info("added path /get_by_attribute/{cid}/{attr_key}/{attr_val:*}")
|
a.log.Info("added path /get_by_attribute/{cid}/{attr_key}/{attr_val:*}")
|
||||||
r.GET("/zip/{cid}/{prefix:*}", a.logger(downloader.DownloadZipped))
|
r.GET("/zip/{cid}/{prefix:*}", a.logger(downloadRoutes.DownloadZipped))
|
||||||
a.log.Info("added path /zip/{cid}/{prefix}")
|
a.log.Info("added path /zip/{cid}/{prefix}")
|
||||||
// enable metrics
|
// enable metrics
|
||||||
if a.cfg.GetBool(cmdMetrics) {
|
if a.cfg.GetBool(cmdMetrics) {
|
||||||
|
@ -267,3 +301,11 @@ func (a *app) logger(h fasthttp.RequestHandler) fasthttp.RequestHandler {
|
||||||
h(ctx)
|
h(ctx)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (a *app) AppParams() *utils.AppParams {
|
||||||
|
return &utils.AppParams{
|
||||||
|
Logger: a.log,
|
||||||
|
Pool: a.pool,
|
||||||
|
Resolver: a.resolver,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
@ -66,6 +66,11 @@ HTTP_GW_STREAM_REQUEST_BODY=true
|
||||||
# The server rejects requests with bodies exceeding this limit.
|
# The server rejects requests with bodies exceeding this limit.
|
||||||
HTTP_GW_MAX_REQUEST_BODY_SIZE=4194304
|
HTTP_GW_MAX_REQUEST_BODY_SIZE=4194304
|
||||||
|
|
||||||
|
# RPC endpoint to be able to use nns container resolving.
|
||||||
|
HTTP_GW_RPC_ENDPOINT=http://morph-chain.neofs.devenv:30333
|
||||||
|
# The order in which resolvers are used to find an container id by name.
|
||||||
|
HTTP_GW_RESOLVE_ORDER="nns dns"
|
||||||
|
|
||||||
# Create timestamp for object if it isn't provided by header.
|
# Create timestamp for object if it isn't provided by header.
|
||||||
HTTP_GW_UPLOAD_HEADER_USE_DEFAULT_TIMESTAMP=false
|
HTTP_GW_UPLOAD_HEADER_USE_DEFAULT_TIMESTAMP=false
|
||||||
|
|
||||||
|
|
|
@ -66,6 +66,12 @@ web:
|
||||||
# The server rejects requests with bodies exceeding this limit.
|
# The server rejects requests with bodies exceeding this limit.
|
||||||
max_request_body_size: 4194304
|
max_request_body_size: 4194304
|
||||||
|
|
||||||
|
# RPC endpoint to be able to use nns container resolving.
|
||||||
|
rpc_endpoint: http://morph-chain.neofs.devenv:30333
|
||||||
|
# The order in which resolvers are used to find an container id by name.
|
||||||
|
resolve_order:
|
||||||
|
- nns
|
||||||
|
- dns
|
||||||
|
|
||||||
upload_header:
|
upload_header:
|
||||||
use_default_timestamp: false # Create timestamp for object if it isn't provided by header.
|
use_default_timestamp: false # Create timestamp for object if it isn't provided by header.
|
||||||
|
|
|
@ -16,6 +16,7 @@ import (
|
||||||
"unicode"
|
"unicode"
|
||||||
"unicode/utf8"
|
"unicode/utf8"
|
||||||
|
|
||||||
|
"github.com/nspcc-dev/neofs-http-gw/resolver"
|
||||||
"github.com/nspcc-dev/neofs-http-gw/response"
|
"github.com/nspcc-dev/neofs-http-gw/response"
|
||||||
"github.com/nspcc-dev/neofs-http-gw/tokens"
|
"github.com/nspcc-dev/neofs-http-gw/tokens"
|
||||||
"github.com/nspcc-dev/neofs-http-gw/utils"
|
"github.com/nspcc-dev/neofs-http-gw/utils"
|
||||||
|
@ -250,10 +251,11 @@ func (r *request) handleNeoFSErr(err error, start time.Time) {
|
||||||
|
|
||||||
// Downloader is a download request handler.
|
// Downloader is a download request handler.
|
||||||
type Downloader struct {
|
type Downloader struct {
|
||||||
appCtx context.Context
|
appCtx context.Context
|
||||||
log *zap.Logger
|
log *zap.Logger
|
||||||
pool *pool.Pool
|
pool *pool.Pool
|
||||||
settings Settings
|
containerResolver *resolver.ContainerResolver
|
||||||
|
settings Settings
|
||||||
}
|
}
|
||||||
|
|
||||||
type Settings struct {
|
type Settings struct {
|
||||||
|
@ -261,8 +263,14 @@ type Settings struct {
|
||||||
}
|
}
|
||||||
|
|
||||||
// New creates an instance of Downloader using specified options.
|
// New creates an instance of Downloader using specified options.
|
||||||
func New(ctx context.Context, log *zap.Logger, settings Settings, conns *pool.Pool) *Downloader {
|
func New(ctx context.Context, params *utils.AppParams, settings Settings) *Downloader {
|
||||||
return &Downloader{appCtx: ctx, log: log, pool: conns, settings: settings}
|
return &Downloader{
|
||||||
|
appCtx: ctx,
|
||||||
|
log: params.Logger,
|
||||||
|
pool: params.Pool,
|
||||||
|
settings: settings,
|
||||||
|
containerResolver: params.Resolver,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (d *Downloader) newRequest(ctx *fasthttp.RequestCtx, log *zap.Logger) *request {
|
func (d *Downloader) newRequest(ctx *fasthttp.RequestCtx, log *zap.Logger) *request {
|
||||||
|
@ -282,18 +290,29 @@ func (d *Downloader) DownloadByAddress(c *fasthttp.RequestCtx) {
|
||||||
// prepares request and object address to it.
|
// prepares request and object address to it.
|
||||||
func (d *Downloader) byAddress(c *fasthttp.RequestCtx, f func(request, *pool.Pool, *address.Address)) {
|
func (d *Downloader) byAddress(c *fasthttp.RequestCtx, f func(request, *pool.Pool, *address.Address)) {
|
||||||
var (
|
var (
|
||||||
addr = address.NewAddress()
|
|
||||||
idCnr, _ = c.UserValue("cid").(string)
|
idCnr, _ = c.UserValue("cid").(string)
|
||||||
idObj, _ = c.UserValue("oid").(string)
|
idObj, _ = c.UserValue("oid").(string)
|
||||||
val = strings.Join([]string{idCnr, idObj}, "/")
|
|
||||||
log = d.log.With(zap.String("cid", idCnr), zap.String("oid", idObj))
|
log = d.log.With(zap.String("cid", idCnr), zap.String("oid", idObj))
|
||||||
)
|
)
|
||||||
if err := addr.Parse(val); err != nil {
|
|
||||||
log.Error("wrong object address", zap.Error(err))
|
cnrID, err := utils.GetContainerID(d.appCtx, idCnr, d.containerResolver)
|
||||||
response.Error(c, "wrong object address", fasthttp.StatusBadRequest)
|
if err != nil {
|
||||||
|
log.Error("wrong container id", zap.Error(err))
|
||||||
|
response.Error(c, "wrong container id", fasthttp.StatusBadRequest)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
objID := new(oid.ID)
|
||||||
|
if err = objID.DecodeString(idObj); err != nil {
|
||||||
|
log.Error("wrong object id", zap.Error(err))
|
||||||
|
response.Error(c, "wrong object id", fasthttp.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
addr := address.NewAddress()
|
||||||
|
addr.SetContainerID(*cnrID)
|
||||||
|
addr.SetObjectID(*objID)
|
||||||
|
|
||||||
f(*d.newRequest(c, log), d.pool, addr)
|
f(*d.newRequest(c, log), d.pool, addr)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -305,16 +324,16 @@ func (d *Downloader) DownloadByAttribute(c *fasthttp.RequestCtx) {
|
||||||
// byAttribute is a wrapper similar to byAddress.
|
// byAttribute is a wrapper similar to byAddress.
|
||||||
func (d *Downloader) byAttribute(c *fasthttp.RequestCtx, f func(request, *pool.Pool, *address.Address)) {
|
func (d *Downloader) byAttribute(c *fasthttp.RequestCtx, f func(request, *pool.Pool, *address.Address)) {
|
||||||
var (
|
var (
|
||||||
httpStatus = fasthttp.StatusBadRequest
|
scid, _ = c.UserValue("cid").(string)
|
||||||
scid, _ = c.UserValue("cid").(string)
|
key, _ = url.QueryUnescape(c.UserValue("attr_key").(string))
|
||||||
key, _ = url.QueryUnescape(c.UserValue("attr_key").(string))
|
val, _ = url.QueryUnescape(c.UserValue("attr_val").(string))
|
||||||
val, _ = url.QueryUnescape(c.UserValue("attr_val").(string))
|
log = d.log.With(zap.String("cid", scid), zap.String("attr_key", key), zap.String("attr_val", val))
|
||||||
log = d.log.With(zap.String("cid", scid), zap.String("attr_key", key), zap.String("attr_val", val))
|
|
||||||
)
|
)
|
||||||
containerID := new(cid.ID)
|
|
||||||
if err := containerID.DecodeString(scid); err != nil {
|
containerID, err := utils.GetContainerID(d.appCtx, scid, d.containerResolver)
|
||||||
|
if err != nil {
|
||||||
log.Error("wrong container id", zap.Error(err))
|
log.Error("wrong container id", zap.Error(err))
|
||||||
response.Error(c, "wrong container id", httpStatus)
|
response.Error(c, "wrong container id", fasthttp.StatusBadRequest)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -370,14 +389,14 @@ func (d *Downloader) DownloadZipped(c *fasthttp.RequestCtx) {
|
||||||
prefix, _ := url.QueryUnescape(c.UserValue("prefix").(string))
|
prefix, _ := url.QueryUnescape(c.UserValue("prefix").(string))
|
||||||
log := d.log.With(zap.String("cid", scid), zap.String("prefix", prefix))
|
log := d.log.With(zap.String("cid", scid), zap.String("prefix", prefix))
|
||||||
|
|
||||||
containerID := new(cid.ID)
|
containerID, err := utils.GetContainerID(d.appCtx, scid, d.containerResolver)
|
||||||
if err := containerID.DecodeString(scid); err != nil {
|
if err != nil {
|
||||||
log.Error("wrong container id", zap.Error(err))
|
log.Error("wrong container id", zap.Error(err))
|
||||||
response.Error(c, "wrong container id", fasthttp.StatusBadRequest)
|
response.Error(c, "wrong container id", fasthttp.StatusBadRequest)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := tokens.StoreBearerToken(c); err != nil {
|
if err = tokens.StoreBearerToken(c); err != nil {
|
||||||
log.Error("could not fetch and store bearer token", zap.Error(err))
|
log.Error("could not fetch and store bearer token", zap.Error(err))
|
||||||
response.Error(c, "could not fetch and store bearer token: "+err.Error(), fasthttp.StatusBadRequest)
|
response.Error(c, "could not fetch and store bearer token: "+err.Error(), fasthttp.StatusBadRequest)
|
||||||
return
|
return
|
||||||
|
|
4
go.mod
4
go.mod
|
@ -42,8 +42,10 @@ require (
|
||||||
github.com/golang/snappy v0.0.4 // indirect
|
github.com/golang/snappy v0.0.4 // indirect
|
||||||
github.com/google/uuid v1.3.0 // indirect
|
github.com/google/uuid v1.3.0 // indirect
|
||||||
github.com/gorilla/mux v1.8.0 // indirect
|
github.com/gorilla/mux v1.8.0 // indirect
|
||||||
|
github.com/gorilla/websocket v1.4.2 // indirect
|
||||||
github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect
|
github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect
|
||||||
github.com/hashicorp/hcl v1.0.0 // indirect
|
github.com/hashicorp/hcl v1.0.0 // indirect
|
||||||
|
github.com/holiman/uint256 v1.2.0 // indirect
|
||||||
github.com/klauspost/compress v1.15.0 // indirect
|
github.com/klauspost/compress v1.15.0 // indirect
|
||||||
github.com/magiconair/properties v1.8.6 // indirect
|
github.com/magiconair/properties v1.8.6 // indirect
|
||||||
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
|
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
|
||||||
|
@ -53,6 +55,7 @@ require (
|
||||||
github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
|
github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
|
||||||
github.com/morikuni/aec v1.0.0 // indirect
|
github.com/morikuni/aec v1.0.0 // indirect
|
||||||
github.com/mr-tron/base58 v1.2.0 // indirect
|
github.com/mr-tron/base58 v1.2.0 // indirect
|
||||||
|
github.com/nspcc-dev/go-ordered-json v0.0.0-20210915112629-e1b6cce73d02 // indirect
|
||||||
github.com/nspcc-dev/hrw v1.0.9 // indirect
|
github.com/nspcc-dev/hrw v1.0.9 // indirect
|
||||||
github.com/nspcc-dev/neofs-crypto v0.3.0 // indirect
|
github.com/nspcc-dev/neofs-crypto v0.3.0 // indirect
|
||||||
github.com/nspcc-dev/rfc6979 v0.2.0 // indirect
|
github.com/nspcc-dev/rfc6979 v0.2.0 // indirect
|
||||||
|
@ -84,6 +87,7 @@ require (
|
||||||
go.uber.org/multierr v1.7.0 // indirect
|
go.uber.org/multierr v1.7.0 // indirect
|
||||||
golang.org/x/crypto v0.0.0-20220214200702-86341886e292 // indirect
|
golang.org/x/crypto v0.0.0-20220214200702-86341886e292 // indirect
|
||||||
golang.org/x/net v0.0.0-20220412020605-290c469a71a5 // indirect
|
golang.org/x/net v0.0.0-20220412020605-290c469a71a5 // indirect
|
||||||
|
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
|
||||||
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad // indirect
|
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad // indirect
|
||||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
|
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
|
||||||
golang.org/x/text v0.3.7 // indirect
|
golang.org/x/text v0.3.7 // indirect
|
||||||
|
|
3
go.sum
3
go.sum
|
@ -549,6 +549,7 @@ github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
|
||||||
github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
|
github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
|
||||||
github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
|
github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
|
||||||
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
|
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
|
||||||
|
github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
|
||||||
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||||
github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
|
github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
|
||||||
github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
|
github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
|
||||||
|
@ -586,6 +587,7 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO
|
||||||
github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
|
github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
|
||||||
github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
|
github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
|
||||||
github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
|
github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
|
||||||
|
github.com/holiman/uint256 v1.2.0 h1:gpSYcPLWGv4sG43I2mVLiDZCNDh/EpGjSk8tmtxitHM=
|
||||||
github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw=
|
github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw=
|
||||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||||
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
|
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
|
||||||
|
@ -979,6 +981,7 @@ github.com/valyala/fasthttp v1.28.0/go.mod h1:cmWIqlu99AO/RKcp1HWaViTqc57FswJOfY
|
||||||
github.com/valyala/fasthttp v1.34.0 h1:d3AAQJ2DRcxJYHm7OXNXtXt2as1vMDfxeIcFvhmGGm4=
|
github.com/valyala/fasthttp v1.34.0 h1:d3AAQJ2DRcxJYHm7OXNXtXt2as1vMDfxeIcFvhmGGm4=
|
||||||
github.com/valyala/fasthttp v1.34.0/go.mod h1:epZA5N+7pY6ZaEKRmstzOuYJx9HI8DI1oaCGZpdH4h0=
|
github.com/valyala/fasthttp v1.34.0/go.mod h1:epZA5N+7pY6ZaEKRmstzOuYJx9HI8DI1oaCGZpdH4h0=
|
||||||
github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
|
github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
|
||||||
|
github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74 h1:JwtAtbp7r/7QSyGz8mKUbYJBg2+6Cd7OjM8o/GNOcVo=
|
||||||
github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74/go.mod h1:RmMWU37GKR2s6pgrIEB4ixgpVCt/cf7dnJv3fuH1J1c=
|
github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74/go.mod h1:RmMWU37GKR2s6pgrIEB4ixgpVCt/cf7dnJv3fuH1J1c=
|
||||||
github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk=
|
github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk=
|
||||||
github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
|
github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
|
||||||
|
|
|
@ -35,10 +35,21 @@ type putResponse struct {
|
||||||
OID string `json:"object_id"`
|
OID string `json:"object_id"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const (
|
||||||
|
testContainerName = "friendly"
|
||||||
|
versionWithNativeNames = "0.27.5"
|
||||||
|
)
|
||||||
|
|
||||||
func TestIntegration(t *testing.T) {
|
func TestIntegration(t *testing.T) {
|
||||||
rootCtx := context.Background()
|
rootCtx := context.Background()
|
||||||
aioImage := "nspccdev/neofs-aio-testcontainer:"
|
aioImage := "nspccdev/neofs-aio-testcontainer:"
|
||||||
versions := []string{"0.24.0", "0.25.1", "0.26.1", "0.27.0", "latest"}
|
versions := []string{
|
||||||
|
"0.24.0",
|
||||||
|
"0.25.1",
|
||||||
|
"0.26.1",
|
||||||
|
"0.27.5",
|
||||||
|
"latest",
|
||||||
|
}
|
||||||
key, err := keys.NewPrivateKeyFromHex("1dd37fba80fec4e6a6f13fd708d8dcb3b29def768017052f6c930fa1c5d90bbb")
|
key, err := keys.NewPrivateKeyFromHex("1dd37fba80fec4e6a6f13fd708d8dcb3b29def768017052f6c930fa1c5d90bbb")
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
@ -48,13 +59,13 @@ func TestIntegration(t *testing.T) {
|
||||||
aioContainer := createDockerContainer(ctx, t, aioImage+version)
|
aioContainer := createDockerContainer(ctx, t, aioImage+version)
|
||||||
cancel := runServer()
|
cancel := runServer()
|
||||||
clientPool := getPool(ctx, t, key)
|
clientPool := getPool(ctx, t, key)
|
||||||
CID, err := createContainer(ctx, t, clientPool)
|
CID, err := createContainer(ctx, t, clientPool, version)
|
||||||
require.NoError(t, err, version)
|
require.NoError(t, err, version)
|
||||||
|
|
||||||
t.Run("simple put "+version, func(t *testing.T) { simplePut(ctx, t, clientPool, CID) })
|
t.Run("simple put "+version, func(t *testing.T) { simplePut(ctx, t, clientPool, CID, version) })
|
||||||
t.Run("simple get "+version, func(t *testing.T) { simpleGet(ctx, t, clientPool, CID) })
|
t.Run("simple get "+version, func(t *testing.T) { simpleGet(ctx, t, clientPool, CID, version) })
|
||||||
t.Run("get by attribute "+version, func(t *testing.T) { getByAttr(ctx, t, clientPool, CID) })
|
t.Run("get by attribute "+version, func(t *testing.T) { getByAttr(ctx, t, clientPool, CID, version) })
|
||||||
t.Run("get zip "+version, func(t *testing.T) { getZip(ctx, t, clientPool, CID) })
|
t.Run("get zip "+version, func(t *testing.T) { getZip(ctx, t, clientPool, CID, version) })
|
||||||
|
|
||||||
cancel()
|
cancel()
|
||||||
err = aioContainer.Terminate(ctx)
|
err = aioContainer.Terminate(ctx)
|
||||||
|
@ -74,10 +85,19 @@ func runServer() context.CancelFunc {
|
||||||
return cancel
|
return cancel
|
||||||
}
|
}
|
||||||
|
|
||||||
func simplePut(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *cid.ID) {
|
func simplePut(ctx context.Context, t *testing.T, p *pool.Pool, CID *cid.ID, version string) {
|
||||||
|
url := "http://localhost:8082/upload/" + CID.String()
|
||||||
|
makePutRequestAndCheck(ctx, t, p, CID, url)
|
||||||
|
|
||||||
|
if version >= versionWithNativeNames {
|
||||||
|
url = "http://localhost:8082/upload/" + testContainerName
|
||||||
|
makePutRequestAndCheck(ctx, t, p, CID, url)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func makePutRequestAndCheck(ctx context.Context, t *testing.T, p *pool.Pool, cnrID *cid.ID, url string) {
|
||||||
content := "content of file"
|
content := "content of file"
|
||||||
keyAttr, valAttr := "User-Attribute", "user value"
|
keyAttr, valAttr := "User-Attribute", "user value"
|
||||||
|
|
||||||
attributes := map[string]string{
|
attributes := map[string]string{
|
||||||
object.AttributeFileName: "newFile.txt",
|
object.AttributeFileName: "newFile.txt",
|
||||||
keyAttr: valAttr,
|
keyAttr: valAttr,
|
||||||
|
@ -92,23 +112,32 @@ func simplePut(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *ci
|
||||||
err = w.Close()
|
err = w.Close()
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
request, err := http.NewRequest(http.MethodPost, "http://localhost:8082/upload/"+CID.String(), &buff)
|
request, err := http.NewRequest(http.MethodPost, url, &buff)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
request.Header.Set("Content-Type", w.FormDataContentType())
|
request.Header.Set("Content-Type", w.FormDataContentType())
|
||||||
request.Header.Set("X-Attribute-"+keyAttr, valAttr)
|
request.Header.Set("X-Attribute-"+keyAttr, valAttr)
|
||||||
|
|
||||||
resp, err := http.DefaultClient.Do(request)
|
resp, err := http.DefaultClient.Do(request)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
defer func() {
|
defer func() {
|
||||||
err = resp.Body.Close()
|
err := resp.Body.Close()
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
}()
|
}()
|
||||||
|
|
||||||
addr := &putResponse{}
|
body, err := io.ReadAll(resp.Body)
|
||||||
err = json.NewDecoder(resp.Body).Decode(addr)
|
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
err = CID.DecodeString(addr.CID)
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
fmt.Println(string(body))
|
||||||
|
}
|
||||||
|
require.Equal(t, http.StatusOK, resp.StatusCode)
|
||||||
|
|
||||||
|
addr := &putResponse{}
|
||||||
|
err = json.Unmarshal(body, addr)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
err = cnrID.DecodeString(addr.CID)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
id := new(oid.ID)
|
id := new(oid.ID)
|
||||||
|
@ -116,7 +145,7 @@ func simplePut(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *ci
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
objectAddress := address.NewAddress()
|
objectAddress := address.NewAddress()
|
||||||
objectAddress.SetContainerID(*CID)
|
objectAddress.SetContainerID(*cnrID)
|
||||||
objectAddress.SetObjectID(*id)
|
objectAddress.SetObjectID(*id)
|
||||||
|
|
||||||
payload := bytes.NewBuffer(nil)
|
payload := bytes.NewBuffer(nil)
|
||||||
|
@ -124,7 +153,7 @@ func simplePut(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *ci
|
||||||
var prm pool.PrmObjectGet
|
var prm pool.PrmObjectGet
|
||||||
prm.SetAddress(*objectAddress)
|
prm.SetAddress(*objectAddress)
|
||||||
|
|
||||||
res, err := clientPool.GetObject(ctx, prm)
|
res, err := p.GetObject(ctx, prm)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
_, err = io.Copy(payload, res.Payload)
|
_, err = io.Copy(payload, res.Payload)
|
||||||
|
@ -137,7 +166,7 @@ func simplePut(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *ci
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func simpleGet(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *cid.ID) {
|
func simpleGet(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *cid.ID, version string) {
|
||||||
content := "content of file"
|
content := "content of file"
|
||||||
attributes := map[string]string{
|
attributes := map[string]string{
|
||||||
"some-attr": "some-get-value",
|
"some-attr": "some-get-value",
|
||||||
|
@ -147,8 +176,18 @@ func simpleGet(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *ci
|
||||||
|
|
||||||
resp, err := http.Get("http://localhost:8082/get/" + CID.String() + "/" + id.String())
|
resp, err := http.Get("http://localhost:8082/get/" + CID.String() + "/" + id.String())
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
checkGetResponse(t, resp, content, attributes)
|
||||||
|
|
||||||
|
if version >= versionWithNativeNames {
|
||||||
|
resp, err = http.Get("http://localhost:8082/get/" + testContainerName + "/" + id.String())
|
||||||
|
require.NoError(t, err)
|
||||||
|
checkGetResponse(t, resp, content, attributes)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func checkGetResponse(t *testing.T, resp *http.Response, content string, attributes map[string]string) {
|
||||||
defer func() {
|
defer func() {
|
||||||
err = resp.Body.Close()
|
err := resp.Body.Close()
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
}()
|
}()
|
||||||
|
|
||||||
|
@ -161,7 +200,7 @@ func simpleGet(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *ci
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func getByAttr(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *cid.ID) {
|
func getByAttr(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *cid.ID, version string) {
|
||||||
keyAttr, valAttr := "some-attr", "some-get-by-attr-value"
|
keyAttr, valAttr := "some-attr", "some-get-by-attr-value"
|
||||||
content := "content of file"
|
content := "content of file"
|
||||||
attributes := map[string]string{keyAttr: valAttr}
|
attributes := map[string]string{keyAttr: valAttr}
|
||||||
|
@ -176,8 +215,18 @@ func getByAttr(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *ci
|
||||||
|
|
||||||
resp, err := http.Get("http://localhost:8082/get_by_attribute/" + CID.String() + "/" + keyAttr + "/" + valAttr)
|
resp, err := http.Get("http://localhost:8082/get_by_attribute/" + CID.String() + "/" + keyAttr + "/" + valAttr)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
checkGetByAttrResponse(t, resp, content, expectedAttr)
|
||||||
|
|
||||||
|
if version >= versionWithNativeNames {
|
||||||
|
resp, err = http.Get("http://localhost:8082/get_by_attribute/" + testContainerName + "/" + keyAttr + "/" + valAttr)
|
||||||
|
require.NoError(t, err)
|
||||||
|
checkGetByAttrResponse(t, resp, content, expectedAttr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func checkGetByAttrResponse(t *testing.T, resp *http.Response, content string, attributes map[string]string) {
|
||||||
defer func() {
|
defer func() {
|
||||||
err = resp.Body.Close()
|
err := resp.Body.Close()
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
}()
|
}()
|
||||||
|
|
||||||
|
@ -185,12 +234,12 @@ func getByAttr(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *ci
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.Equal(t, content, string(data))
|
require.Equal(t, content, string(data))
|
||||||
|
|
||||||
for k, v := range expectedAttr {
|
for k, v := range attributes {
|
||||||
require.Equal(t, v, resp.Header.Get(k))
|
require.Equal(t, v, resp.Header.Get(k))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func getZip(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *cid.ID) {
|
func getZip(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *cid.ID, version string) {
|
||||||
names := []string{"zipfolder/dir/name1.txt", "zipfolder/name2.txt"}
|
names := []string{"zipfolder/dir/name1.txt", "zipfolder/name2.txt"}
|
||||||
contents := []string{"content of file1", "content of file2"}
|
contents := []string{"content of file1", "content of file2"}
|
||||||
attributes1 := map[string]string{attributeFilePath: names[0]}
|
attributes1 := map[string]string{attributeFilePath: names[0]}
|
||||||
|
@ -199,28 +248,35 @@ func getZip(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *cid.I
|
||||||
putObject(ctx, t, clientPool, CID, contents[0], attributes1)
|
putObject(ctx, t, clientPool, CID, contents[0], attributes1)
|
||||||
putObject(ctx, t, clientPool, CID, contents[1], attributes2)
|
putObject(ctx, t, clientPool, CID, contents[1], attributes2)
|
||||||
|
|
||||||
resp, err := http.Get("http://localhost:8082/zip/" + CID.String() + "/zipfolder")
|
baseURL := "http://localhost:8082/zip/" + CID.String()
|
||||||
|
makeZipTest(t, baseURL, names, contents)
|
||||||
|
|
||||||
|
if version >= versionWithNativeNames {
|
||||||
|
baseURL = "http://localhost:8082/zip/" + testContainerName
|
||||||
|
makeZipTest(t, baseURL, names, contents)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func makeZipTest(t *testing.T, baseURL string, names, contents []string) {
|
||||||
|
url := baseURL + "/zipfolder"
|
||||||
|
makeZipRequest(t, url, names, contents)
|
||||||
|
|
||||||
|
// check nested folder
|
||||||
|
url = baseURL + "/zipfolder/dir"
|
||||||
|
makeZipRequest(t, url, names[:1], contents[:1])
|
||||||
|
}
|
||||||
|
|
||||||
|
func makeZipRequest(t *testing.T, url string, names, contents []string) {
|
||||||
|
resp, err := http.Get(url)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
defer func() {
|
defer func() {
|
||||||
err = resp.Body.Close()
|
err := resp.Body.Close()
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
}()
|
}()
|
||||||
|
|
||||||
data, err := io.ReadAll(resp.Body)
|
data, err := io.ReadAll(resp.Body)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
checkZip(t, data, resp.ContentLength, names, contents)
|
checkZip(t, data, resp.ContentLength, names, contents)
|
||||||
|
|
||||||
// check nested folder
|
|
||||||
resp2, err := http.Get("http://localhost:8082/zip/" + CID.String() + "/zipfolder/dir")
|
|
||||||
require.NoError(t, err)
|
|
||||||
defer func() {
|
|
||||||
err = resp2.Body.Close()
|
|
||||||
require.NoError(t, err)
|
|
||||||
}()
|
|
||||||
|
|
||||||
data2, err := io.ReadAll(resp2.Body)
|
|
||||||
require.NoError(t, err)
|
|
||||||
checkZip(t, data2, resp2.ContentLength, names[:1], contents[:1])
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func checkZip(t *testing.T, data []byte, length int64, names, contents []string) {
|
func checkZip(t *testing.T, data []byte, length int64, names, contents []string) {
|
||||||
|
@ -273,6 +329,8 @@ func getDefaultConfig() *viper.Viper {
|
||||||
v.SetDefault(cfgPeers+".0.weight", 1)
|
v.SetDefault(cfgPeers+".0.weight", 1)
|
||||||
v.SetDefault(cfgPeers+".0.priority", 1)
|
v.SetDefault(cfgPeers+".0.priority", 1)
|
||||||
|
|
||||||
|
v.SetDefault(cfgRPCEndpoint, "http://127.0.0.1:30333")
|
||||||
|
|
||||||
return v
|
return v
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -290,17 +348,20 @@ func getPool(ctx context.Context, t *testing.T, key *keys.PrivateKey) *pool.Pool
|
||||||
return clientPool
|
return clientPool
|
||||||
}
|
}
|
||||||
|
|
||||||
func createContainer(ctx context.Context, t *testing.T, clientPool *pool.Pool) (*cid.ID, error) {
|
func createContainer(ctx context.Context, t *testing.T, clientPool *pool.Pool, version string) (*cid.ID, error) {
|
||||||
pp, err := policy.Parse("REP 1")
|
pp, err := policy.Parse("REP 1")
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
cnr := container.New(
|
cnr := container.New(
|
||||||
container.WithPolicy(pp),
|
container.WithPolicy(pp),
|
||||||
container.WithCustomBasicACL(0x0FFFFFFF),
|
container.WithCustomBasicACL(0x0FFFFFFF),
|
||||||
container.WithAttribute(container.AttributeName, "friendlyName"),
|
|
||||||
container.WithAttribute(container.AttributeTimestamp, strconv.FormatInt(time.Now().Unix(), 10)))
|
container.WithAttribute(container.AttributeTimestamp, strconv.FormatInt(time.Now().Unix(), 10)))
|
||||||
cnr.SetOwnerID(clientPool.OwnerID())
|
cnr.SetOwnerID(clientPool.OwnerID())
|
||||||
|
|
||||||
|
if version >= versionWithNativeNames {
|
||||||
|
container.SetNativeName(cnr, testContainerName)
|
||||||
|
}
|
||||||
|
|
||||||
var waitPrm pool.WaitParams
|
var waitPrm pool.WaitParams
|
||||||
waitPrm.SetTimeout(15 * time.Second)
|
waitPrm.SetTimeout(15 * time.Second)
|
||||||
waitPrm.SetPollInterval(3 * time.Second)
|
waitPrm.SetPollInterval(3 * time.Second)
|
||||||
|
|
46
resolver/neofs.go
Normal file
46
resolver/neofs.go
Normal file
|
@ -0,0 +1,46 @@
|
||||||
|
package resolver
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/nspcc-dev/neofs-sdk-go/netmap"
|
||||||
|
"github.com/nspcc-dev/neofs-sdk-go/pool"
|
||||||
|
)
|
||||||
|
|
||||||
|
// NeoFSResolver represents virtual connection to the NeoFS network.
|
||||||
|
// It implements resolver.NeoFS.
|
||||||
|
type NeoFSResolver struct {
|
||||||
|
pool *pool.Pool
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewNeoFSResolver creates new NeoFSResolver using provided pool.Pool.
|
||||||
|
func NewNeoFSResolver(p *pool.Pool) *NeoFSResolver {
|
||||||
|
return &NeoFSResolver{pool: p}
|
||||||
|
}
|
||||||
|
|
||||||
|
// SystemDNS implements resolver.NeoFS interface method.
|
||||||
|
func (x *NeoFSResolver) SystemDNS(ctx context.Context) (string, error) {
|
||||||
|
networkInfo, err := x.pool.NetworkInfo(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("read network info via client: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var domain string
|
||||||
|
|
||||||
|
networkInfo.NetworkConfig().IterateParameters(func(parameter *netmap.NetworkParameter) bool {
|
||||||
|
if string(parameter.Key()) == "SystemDNS" {
|
||||||
|
domain = string(parameter.Value())
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
return false
|
||||||
|
})
|
||||||
|
|
||||||
|
if domain == "" {
|
||||||
|
return "", errors.New("system DNS parameter not found or empty")
|
||||||
|
}
|
||||||
|
|
||||||
|
return domain, nil
|
||||||
|
}
|
140
resolver/resolver.go
Normal file
140
resolver/resolver.go
Normal file
|
@ -0,0 +1,140 @@
|
||||||
|
package resolver
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
|
||||||
|
"github.com/nspcc-dev/neofs-sdk-go/ns"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
NNSResolver = "nns"
|
||||||
|
DNSResolver = "dns"
|
||||||
|
)
|
||||||
|
|
||||||
|
// NeoFS represents virtual connection to the NeoFS network.
|
||||||
|
type NeoFS interface {
|
||||||
|
// SystemDNS reads system DNS network parameters of the NeoFS.
|
||||||
|
//
|
||||||
|
// Returns exactly on non-zero value. Returns any error encountered
|
||||||
|
// which prevented the parameter to be read.
|
||||||
|
SystemDNS(context.Context) (string, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
type Config struct {
|
||||||
|
NeoFS NeoFS
|
||||||
|
RPCAddress string
|
||||||
|
}
|
||||||
|
|
||||||
|
type ContainerResolver struct {
|
||||||
|
Name string
|
||||||
|
resolve func(context.Context, string) (*cid.ID, error)
|
||||||
|
|
||||||
|
next *ContainerResolver
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *ContainerResolver) SetResolveFunc(fn func(context.Context, string) (*cid.ID, error)) {
|
||||||
|
r.resolve = fn
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *ContainerResolver) Resolve(ctx context.Context, name string) (*cid.ID, error) {
|
||||||
|
cnrID, err := r.resolve(ctx, name)
|
||||||
|
if err != nil {
|
||||||
|
if r.next != nil {
|
||||||
|
cnrID, inErr := r.next.Resolve(ctx, name)
|
||||||
|
if inErr != nil {
|
||||||
|
return nil, fmt.Errorf("%s; %w", err.Error(), inErr)
|
||||||
|
}
|
||||||
|
return cnrID, nil
|
||||||
|
}
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return cnrID, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewResolver(order []string, cfg *Config) (*ContainerResolver, error) {
|
||||||
|
if len(order) == 0 {
|
||||||
|
return nil, fmt.Errorf("resolving order must not be empty")
|
||||||
|
}
|
||||||
|
|
||||||
|
bucketResolver, err := newResolver(order[len(order)-1], cfg, nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
for i := len(order) - 2; i >= 0; i-- {
|
||||||
|
resolverName := order[i]
|
||||||
|
next := bucketResolver
|
||||||
|
|
||||||
|
bucketResolver, err = newResolver(resolverName, cfg, next)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return bucketResolver, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func newResolver(name string, cfg *Config, next *ContainerResolver) (*ContainerResolver, error) {
|
||||||
|
switch name {
|
||||||
|
case DNSResolver:
|
||||||
|
return NewDNSResolver(cfg.NeoFS, next)
|
||||||
|
case NNSResolver:
|
||||||
|
return NewNNSResolver(cfg.RPCAddress, next)
|
||||||
|
default:
|
||||||
|
return nil, fmt.Errorf("unknown resolver: %s", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewDNSResolver(neoFS NeoFS, next *ContainerResolver) (*ContainerResolver, error) {
|
||||||
|
if neoFS == nil {
|
||||||
|
return nil, fmt.Errorf("pool must not be nil for DNS resolver")
|
||||||
|
}
|
||||||
|
|
||||||
|
var dns ns.DNS
|
||||||
|
|
||||||
|
resolveFunc := func(ctx context.Context, name string) (*cid.ID, error) {
|
||||||
|
domain, err := neoFS.SystemDNS(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("read system DNS parameter of the NeoFS: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
domain = name + "." + domain
|
||||||
|
cnrID, err := dns.ResolveContainerName(domain)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("couldn't resolve container '%s' as '%s': %w", name, domain, err)
|
||||||
|
}
|
||||||
|
return &cnrID, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return &ContainerResolver{
|
||||||
|
Name: DNSResolver,
|
||||||
|
|
||||||
|
resolve: resolveFunc,
|
||||||
|
next: next,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewNNSResolver(rpcAddress string, next *ContainerResolver) (*ContainerResolver, error) {
|
||||||
|
var nns ns.NNS
|
||||||
|
|
||||||
|
if err := nns.Dial(rpcAddress); err != nil {
|
||||||
|
return nil, fmt.Errorf("could not dial nns: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
resolveFunc := func(_ context.Context, name string) (*cid.ID, error) {
|
||||||
|
cnrID, err := nns.ResolveContainerName(name)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("couldn't resolve container '%s': %w", name, err)
|
||||||
|
}
|
||||||
|
return &cnrID, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return &ContainerResolver{
|
||||||
|
Name: NNSResolver,
|
||||||
|
|
||||||
|
resolve: resolveFunc,
|
||||||
|
next: next,
|
||||||
|
}, nil
|
||||||
|
}
|
13
settings.go
13
settings.go
|
@ -8,6 +8,7 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/nspcc-dev/neofs-http-gw/resolver"
|
||||||
"github.com/spf13/pflag"
|
"github.com/spf13/pflag"
|
||||||
"github.com/spf13/viper"
|
"github.com/spf13/viper"
|
||||||
"github.com/valyala/fasthttp"
|
"github.com/valyala/fasthttp"
|
||||||
|
@ -49,6 +50,12 @@ const (
|
||||||
// Peers.
|
// Peers.
|
||||||
cfgPeers = "peers"
|
cfgPeers = "peers"
|
||||||
|
|
||||||
|
// NeoGo.
|
||||||
|
cfgRPCEndpoint = "rpc_endpoint"
|
||||||
|
|
||||||
|
// Resolving.
|
||||||
|
cfgResolveOrder = "resolve_order"
|
||||||
|
|
||||||
// Zip compression.
|
// Zip compression.
|
||||||
cfgZipCompression = "zip.compression"
|
cfgZipCompression = "zip.compression"
|
||||||
|
|
||||||
|
@ -99,6 +106,8 @@ func settings() *viper.Viper {
|
||||||
flags.String(cfgTLSKey, "", "TLS key path")
|
flags.String(cfgTLSKey, "", "TLS key path")
|
||||||
peers := flags.StringArrayP(cfgPeers, "p", nil, "NeoFS nodes")
|
peers := flags.StringArrayP(cfgPeers, "p", nil, "NeoFS nodes")
|
||||||
|
|
||||||
|
resolveMethods := flags.StringSlice(cfgResolveOrder, []string{resolver.NNSResolver, resolver.DNSResolver}, "set container name resolve order")
|
||||||
|
|
||||||
// set defaults:
|
// set defaults:
|
||||||
|
|
||||||
// logger:
|
// logger:
|
||||||
|
@ -126,6 +135,10 @@ func settings() *viper.Viper {
|
||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if resolveMethods != nil {
|
||||||
|
v.SetDefault(cfgResolveOrder, *resolveMethods)
|
||||||
|
}
|
||||||
|
|
||||||
switch {
|
switch {
|
||||||
case help != nil && *help:
|
case help != nil && *help:
|
||||||
fmt.Printf("NeoFS HTTP Gateway %s\n", Version)
|
fmt.Printf("NeoFS HTTP Gateway %s\n", Version)
|
||||||
|
|
|
@ -9,11 +9,11 @@ import (
|
||||||
"strconv"
|
"strconv"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/nspcc-dev/neofs-http-gw/resolver"
|
||||||
"github.com/nspcc-dev/neofs-http-gw/response"
|
"github.com/nspcc-dev/neofs-http-gw/response"
|
||||||
"github.com/nspcc-dev/neofs-http-gw/tokens"
|
"github.com/nspcc-dev/neofs-http-gw/tokens"
|
||||||
"github.com/nspcc-dev/neofs-http-gw/utils"
|
"github.com/nspcc-dev/neofs-http-gw/utils"
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/bearer"
|
"github.com/nspcc-dev/neofs-sdk-go/bearer"
|
||||||
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
|
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/netmap"
|
"github.com/nspcc-dev/neofs-sdk-go/netmap"
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/object"
|
"github.com/nspcc-dev/neofs-sdk-go/object"
|
||||||
"github.com/nspcc-dev/neofs-sdk-go/object/address"
|
"github.com/nspcc-dev/neofs-sdk-go/object/address"
|
||||||
|
@ -35,6 +35,7 @@ type Uploader struct {
|
||||||
log *zap.Logger
|
log *zap.Logger
|
||||||
pool *pool.Pool
|
pool *pool.Pool
|
||||||
enableDefaultTimestamp bool
|
enableDefaultTimestamp bool
|
||||||
|
containerResolver *resolver.ContainerResolver
|
||||||
}
|
}
|
||||||
|
|
||||||
type epochDurations struct {
|
type epochDurations struct {
|
||||||
|
@ -45,33 +46,41 @@ type epochDurations struct {
|
||||||
|
|
||||||
// New creates a new Uploader using specified logger, connection pool and
|
// New creates a new Uploader using specified logger, connection pool and
|
||||||
// other options.
|
// other options.
|
||||||
func New(ctx context.Context, log *zap.Logger, conns *pool.Pool, enableDefaultTimestamp bool) *Uploader {
|
func New(ctx context.Context, params *utils.AppParams, enableDefaultTimestamp bool) *Uploader {
|
||||||
return &Uploader{ctx, log, conns, enableDefaultTimestamp}
|
return &Uploader{
|
||||||
|
appCtx: ctx,
|
||||||
|
log: params.Logger,
|
||||||
|
pool: params.Pool,
|
||||||
|
enableDefaultTimestamp: enableDefaultTimestamp,
|
||||||
|
containerResolver: params.Resolver,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Upload handles multipart upload request.
|
// Upload handles multipart upload request.
|
||||||
func (u *Uploader) Upload(c *fasthttp.RequestCtx) {
|
func (u *Uploader) Upload(c *fasthttp.RequestCtx) {
|
||||||
var (
|
var (
|
||||||
err error
|
|
||||||
file MultipartFile
|
file MultipartFile
|
||||||
idObj *oid.ID
|
idObj *oid.ID
|
||||||
addr = address.NewAddress()
|
addr = address.NewAddress()
|
||||||
idCnr = new(cid.ID)
|
|
||||||
scid, _ = c.UserValue("cid").(string)
|
scid, _ = c.UserValue("cid").(string)
|
||||||
log = u.log.With(zap.String("cid", scid))
|
log = u.log.With(zap.String("cid", scid))
|
||||||
bodyStream = c.RequestBodyStream()
|
bodyStream = c.RequestBodyStream()
|
||||||
drainBuf = make([]byte, drainBufSize)
|
drainBuf = make([]byte, drainBufSize)
|
||||||
)
|
)
|
||||||
if err = tokens.StoreBearerToken(c); err != nil {
|
|
||||||
|
if err := tokens.StoreBearerToken(c); err != nil {
|
||||||
log.Error("could not fetch bearer token", zap.Error(err))
|
log.Error("could not fetch bearer token", zap.Error(err))
|
||||||
response.Error(c, "could not fetch bearer token", fasthttp.StatusBadRequest)
|
response.Error(c, "could not fetch bearer token", fasthttp.StatusBadRequest)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if err = idCnr.DecodeString(scid); err != nil {
|
|
||||||
|
idCnr, err := utils.GetContainerID(u.appCtx, scid, u.containerResolver)
|
||||||
|
if err != nil {
|
||||||
log.Error("wrong container id", zap.Error(err))
|
log.Error("wrong container id", zap.Error(err))
|
||||||
response.Error(c, "wrong container id", fasthttp.StatusBadRequest)
|
response.Error(c, "wrong container id", fasthttp.StatusBadRequest)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
defer func() {
|
defer func() {
|
||||||
// If the temporary reader can be closed - let's close it.
|
// If the temporary reader can be closed - let's close it.
|
||||||
if file == nil {
|
if file == nil {
|
||||||
|
|
13
utils/params.go
Normal file
13
utils/params.go
Normal file
|
@ -0,0 +1,13 @@
|
||||||
|
package utils
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/nspcc-dev/neofs-http-gw/resolver"
|
||||||
|
"github.com/nspcc-dev/neofs-sdk-go/pool"
|
||||||
|
"go.uber.org/zap"
|
||||||
|
)
|
||||||
|
|
||||||
|
type AppParams struct {
|
||||||
|
Logger *zap.Logger
|
||||||
|
Pool *pool.Pool
|
||||||
|
Resolver *resolver.ContainerResolver
|
||||||
|
}
|
21
utils/util.go
Normal file
21
utils/util.go
Normal file
|
@ -0,0 +1,21 @@
|
||||||
|
package utils
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
|
||||||
|
"github.com/nspcc-dev/neofs-http-gw/resolver"
|
||||||
|
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
|
||||||
|
)
|
||||||
|
|
||||||
|
// GetContainerID decode container id, if it's not a valid container id
|
||||||
|
// then trey to resolve name using provided resolver.
|
||||||
|
func GetContainerID(ctx context.Context, containerID string, resolver *resolver.ContainerResolver) (*cid.ID, error) {
|
||||||
|
cnrID := new(cid.ID)
|
||||||
|
err := cnrID.DecodeString(containerID)
|
||||||
|
if err != nil {
|
||||||
|
if resolver != nil {
|
||||||
|
cnrID, err = resolver.Resolve(ctx, containerID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return cnrID, err
|
||||||
|
}
|
Loading…
Reference in a new issue