From a42606742a5360523a1098abb04ed3ff15f1fdb3 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Wed, 20 Apr 2022 12:17:20 +0300 Subject: [PATCH] [#142] Support resolving container nicename Signed-off-by: Denis Kirillov --- README.md | 2 + app.go | 58 ++++++++++++++--- config/config.env | 5 ++ config/config.yaml | 6 ++ downloader/download.go | 63 ++++++++++++------- go.mod | 4 ++ go.sum | 3 + integration_test.go | 135 ++++++++++++++++++++++++++++----------- resolver/neofs.go | 46 ++++++++++++++ resolver/resolver.go | 140 +++++++++++++++++++++++++++++++++++++++++ settings.go | 13 ++++ uploader/upload.go | 23 ++++--- utils/params.go | 13 ++++ utils/util.go | 21 +++++++ 14 files changed, 458 insertions(+), 74 deletions(-) create mode 100644 resolver/neofs.go create mode 100644 resolver/resolver.go create mode 100644 utils/params.go create mode 100644 utils/util.go diff --git a/README.md b/README.md index 563fb26..94aa369 100644 --- a/README.md +++ b/README.md @@ -189,6 +189,8 @@ and upload objects with it, but deleting, searching, managing ACLs, creating containers and other activities are not supported and not planned to be supported. +**Note:** in all download/upload routes you can use container name instead of it's id (`$CID`), but resolvers must be configured properly (see [configs](./config) for examples). + ### Preparation Before uploading or downloading a file make sure you have a prepared container. diff --git a/app.go b/app.go index bfdd66b..26201c6 100644 --- a/app.go +++ b/app.go @@ -13,8 +13,10 @@ import ( "github.com/nspcc-dev/neo-go/pkg/util" "github.com/nspcc-dev/neo-go/pkg/wallet" "github.com/nspcc-dev/neofs-http-gw/downloader" + "github.com/nspcc-dev/neofs-http-gw/resolver" "github.com/nspcc-dev/neofs-http-gw/response" "github.com/nspcc-dev/neofs-http-gw/uploader" + "github.com/nspcc-dev/neofs-http-gw/utils" "github.com/nspcc-dev/neofs-sdk-go/pool" "github.com/spf13/viper" "github.com/valyala/fasthttp" @@ -28,6 +30,7 @@ type ( cfg *viper.Viper webServer *fasthttp.Server webDone chan struct{} + resolver *resolver.ContainerResolver } // App is an interface for the main gateway function. @@ -127,9 +130,39 @@ func newApp(ctx context.Context, opt ...Option) App { if err != nil { a.log.Fatal("failed to dial pool", zap.Error(err)) } + + resolveCfg := &resolver.Config{ + NeoFS: resolver.NewNeoFSResolver(a.pool), + RPCAddress: a.cfg.GetString(cfgRPCEndpoint), + } + + order := a.cfg.GetStringSlice(cfgResolveOrder) + if resolveCfg.RPCAddress == "" { + order = remove(order, resolver.NNSResolver) + a.log.Warn(fmt.Sprintf("resolver '%s' won't be used since '%s' isn't provided", resolver.NNSResolver, cfgRPCEndpoint)) + } + + if len(order) != 0 { + a.resolver, err = resolver.NewResolver(order, resolveCfg) + if err != nil { + a.log.Fatal("failed to create resolver", zap.Error(err)) + } + } else { + a.log.Info("container resolver is disabled") + } + return a } +func remove(list []string, element string) []string { + for i, item := range list { + if item == element { + return append(list[:i], list[i+1:]...) + } + } + return list +} + func getNeoFSKey(a *app) (*ecdsa.PrivateKey, error) { walletPath := a.cfg.GetString(cmdWallet) if len(walletPath) == 0 { @@ -208,8 +241,9 @@ func (a *app) Serve(ctx context.Context) { close(a.webDone) }() edts := a.cfg.GetBool(cfgUploaderHeaderEnableDefaultTimestamp) - uploader := uploader.New(ctx, a.log, a.pool, edts) - downloader := downloader.New(ctx, a.log, downloader.Settings{ZipCompression: a.cfg.GetBool(cfgZipCompression)}, a.pool) + uploadRoutes := uploader.New(ctx, a.AppParams(), edts) + downloadSettings := downloader.Settings{ZipCompression: a.cfg.GetBool(cfgZipCompression)} + downloadRoutes := downloader.New(ctx, a.AppParams(), downloadSettings) // Configure router. r := router.New() r.RedirectTrailingSlash = true @@ -219,15 +253,15 @@ func (a *app) Serve(ctx context.Context) { r.MethodNotAllowed = func(r *fasthttp.RequestCtx) { response.Error(r, "Method Not Allowed", fasthttp.StatusMethodNotAllowed) } - r.POST("/upload/{cid}", a.logger(uploader.Upload)) + r.POST("/upload/{cid}", a.logger(uploadRoutes.Upload)) a.log.Info("added path /upload/{cid}") - r.GET("/get/{cid}/{oid}", a.logger(downloader.DownloadByAddress)) - r.HEAD("/get/{cid}/{oid}", a.logger(downloader.HeadByAddress)) + r.GET("/get/{cid}/{oid}", a.logger(downloadRoutes.DownloadByAddress)) + r.HEAD("/get/{cid}/{oid}", a.logger(downloadRoutes.HeadByAddress)) a.log.Info("added path /get/{cid}/{oid}") - r.GET("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.logger(downloader.DownloadByAttribute)) - r.HEAD("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.logger(downloader.HeadByAttribute)) + r.GET("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.logger(downloadRoutes.DownloadByAttribute)) + r.HEAD("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.logger(downloadRoutes.HeadByAttribute)) a.log.Info("added path /get_by_attribute/{cid}/{attr_key}/{attr_val:*}") - r.GET("/zip/{cid}/{prefix:*}", a.logger(downloader.DownloadZipped)) + r.GET("/zip/{cid}/{prefix:*}", a.logger(downloadRoutes.DownloadZipped)) a.log.Info("added path /zip/{cid}/{prefix}") // enable metrics if a.cfg.GetBool(cmdMetrics) { @@ -267,3 +301,11 @@ func (a *app) logger(h fasthttp.RequestHandler) fasthttp.RequestHandler { h(ctx) }) } + +func (a *app) AppParams() *utils.AppParams { + return &utils.AppParams{ + Logger: a.log, + Pool: a.pool, + Resolver: a.resolver, + } +} diff --git a/config/config.env b/config/config.env index 564b2dd..a6bd656 100644 --- a/config/config.env +++ b/config/config.env @@ -66,6 +66,11 @@ HTTP_GW_STREAM_REQUEST_BODY=true # The server rejects requests with bodies exceeding this limit. HTTP_GW_MAX_REQUEST_BODY_SIZE=4194304 +# RPC endpoint to be able to use nns container resolving. +HTTP_GW_RPC_ENDPOINT=http://morph-chain.neofs.devenv:30333 +# The order in which resolvers are used to find an container id by name. +HTTP_GW_RESOLVE_ORDER="nns dns" + # Create timestamp for object if it isn't provided by header. HTTP_GW_UPLOAD_HEADER_USE_DEFAULT_TIMESTAMP=false diff --git a/config/config.yaml b/config/config.yaml index 7cf3e5f..9cdb7ce 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -66,6 +66,12 @@ web: # The server rejects requests with bodies exceeding this limit. max_request_body_size: 4194304 +# RPC endpoint to be able to use nns container resolving. +rpc_endpoint: http://morph-chain.neofs.devenv:30333 +# The order in which resolvers are used to find an container id by name. +resolve_order: + - nns + - dns upload_header: use_default_timestamp: false # Create timestamp for object if it isn't provided by header. diff --git a/downloader/download.go b/downloader/download.go index 542e85d..5456f91 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -16,6 +16,7 @@ import ( "unicode" "unicode/utf8" + "github.com/nspcc-dev/neofs-http-gw/resolver" "github.com/nspcc-dev/neofs-http-gw/response" "github.com/nspcc-dev/neofs-http-gw/tokens" "github.com/nspcc-dev/neofs-http-gw/utils" @@ -250,10 +251,11 @@ func (r *request) handleNeoFSErr(err error, start time.Time) { // Downloader is a download request handler. type Downloader struct { - appCtx context.Context - log *zap.Logger - pool *pool.Pool - settings Settings + appCtx context.Context + log *zap.Logger + pool *pool.Pool + containerResolver *resolver.ContainerResolver + settings Settings } type Settings struct { @@ -261,8 +263,14 @@ type Settings struct { } // New creates an instance of Downloader using specified options. -func New(ctx context.Context, log *zap.Logger, settings Settings, conns *pool.Pool) *Downloader { - return &Downloader{appCtx: ctx, log: log, pool: conns, settings: settings} +func New(ctx context.Context, params *utils.AppParams, settings Settings) *Downloader { + return &Downloader{ + appCtx: ctx, + log: params.Logger, + pool: params.Pool, + settings: settings, + containerResolver: params.Resolver, + } } func (d *Downloader) newRequest(ctx *fasthttp.RequestCtx, log *zap.Logger) *request { @@ -282,18 +290,29 @@ func (d *Downloader) DownloadByAddress(c *fasthttp.RequestCtx) { // prepares request and object address to it. func (d *Downloader) byAddress(c *fasthttp.RequestCtx, f func(request, *pool.Pool, *address.Address)) { var ( - addr = address.NewAddress() idCnr, _ = c.UserValue("cid").(string) idObj, _ = c.UserValue("oid").(string) - val = strings.Join([]string{idCnr, idObj}, "/") log = d.log.With(zap.String("cid", idCnr), zap.String("oid", idObj)) ) - if err := addr.Parse(val); err != nil { - log.Error("wrong object address", zap.Error(err)) - response.Error(c, "wrong object address", fasthttp.StatusBadRequest) + + cnrID, err := utils.GetContainerID(d.appCtx, idCnr, d.containerResolver) + if err != nil { + log.Error("wrong container id", zap.Error(err)) + response.Error(c, "wrong container id", fasthttp.StatusBadRequest) return } + objID := new(oid.ID) + if err = objID.DecodeString(idObj); err != nil { + log.Error("wrong object id", zap.Error(err)) + response.Error(c, "wrong object id", fasthttp.StatusBadRequest) + return + } + + addr := address.NewAddress() + addr.SetContainerID(*cnrID) + addr.SetObjectID(*objID) + f(*d.newRequest(c, log), d.pool, addr) } @@ -305,16 +324,16 @@ func (d *Downloader) DownloadByAttribute(c *fasthttp.RequestCtx) { // byAttribute is a wrapper similar to byAddress. func (d *Downloader) byAttribute(c *fasthttp.RequestCtx, f func(request, *pool.Pool, *address.Address)) { var ( - httpStatus = fasthttp.StatusBadRequest - scid, _ = c.UserValue("cid").(string) - key, _ = url.QueryUnescape(c.UserValue("attr_key").(string)) - val, _ = url.QueryUnescape(c.UserValue("attr_val").(string)) - log = d.log.With(zap.String("cid", scid), zap.String("attr_key", key), zap.String("attr_val", val)) + scid, _ = c.UserValue("cid").(string) + key, _ = url.QueryUnescape(c.UserValue("attr_key").(string)) + val, _ = url.QueryUnescape(c.UserValue("attr_val").(string)) + log = d.log.With(zap.String("cid", scid), zap.String("attr_key", key), zap.String("attr_val", val)) ) - containerID := new(cid.ID) - if err := containerID.DecodeString(scid); err != nil { + + containerID, err := utils.GetContainerID(d.appCtx, scid, d.containerResolver) + if err != nil { log.Error("wrong container id", zap.Error(err)) - response.Error(c, "wrong container id", httpStatus) + response.Error(c, "wrong container id", fasthttp.StatusBadRequest) return } @@ -370,14 +389,14 @@ func (d *Downloader) DownloadZipped(c *fasthttp.RequestCtx) { prefix, _ := url.QueryUnescape(c.UserValue("prefix").(string)) log := d.log.With(zap.String("cid", scid), zap.String("prefix", prefix)) - containerID := new(cid.ID) - if err := containerID.DecodeString(scid); err != nil { + containerID, err := utils.GetContainerID(d.appCtx, scid, d.containerResolver) + if err != nil { log.Error("wrong container id", zap.Error(err)) response.Error(c, "wrong container id", fasthttp.StatusBadRequest) return } - if err := tokens.StoreBearerToken(c); err != nil { + if err = tokens.StoreBearerToken(c); err != nil { log.Error("could not fetch and store bearer token", zap.Error(err)) response.Error(c, "could not fetch and store bearer token: "+err.Error(), fasthttp.StatusBadRequest) return diff --git a/go.mod b/go.mod index f2b86b7..bf5bf37 100644 --- a/go.mod +++ b/go.mod @@ -42,8 +42,10 @@ require ( github.com/golang/snappy v0.0.4 // indirect github.com/google/uuid v1.3.0 // indirect github.com/gorilla/mux v1.8.0 // indirect + github.com/gorilla/websocket v1.4.2 // indirect github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect github.com/hashicorp/hcl v1.0.0 // indirect + github.com/holiman/uint256 v1.2.0 // indirect github.com/klauspost/compress v1.15.0 // indirect github.com/magiconair/properties v1.8.6 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect @@ -53,6 +55,7 @@ require ( github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect github.com/morikuni/aec v1.0.0 // indirect github.com/mr-tron/base58 v1.2.0 // indirect + github.com/nspcc-dev/go-ordered-json v0.0.0-20210915112629-e1b6cce73d02 // indirect github.com/nspcc-dev/hrw v1.0.9 // indirect github.com/nspcc-dev/neofs-crypto v0.3.0 // indirect github.com/nspcc-dev/rfc6979 v0.2.0 // indirect @@ -84,6 +87,7 @@ require ( go.uber.org/multierr v1.7.0 // indirect golang.org/x/crypto v0.0.0-20220214200702-86341886e292 // indirect golang.org/x/net v0.0.0-20220412020605-290c469a71a5 // indirect + golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect golang.org/x/sys v0.0.0-20220412211240-33da011f77ad // indirect golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect golang.org/x/text v0.3.7 // indirect diff --git a/go.sum b/go.sum index c869b51..0ee3fb9 100644 --- a/go.sum +++ b/go.sum @@ -549,6 +549,7 @@ github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= @@ -586,6 +587,7 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/holiman/uint256 v1.2.0 h1:gpSYcPLWGv4sG43I2mVLiDZCNDh/EpGjSk8tmtxitHM= github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -979,6 +981,7 @@ github.com/valyala/fasthttp v1.28.0/go.mod h1:cmWIqlu99AO/RKcp1HWaViTqc57FswJOfY github.com/valyala/fasthttp v1.34.0 h1:d3AAQJ2DRcxJYHm7OXNXtXt2as1vMDfxeIcFvhmGGm4= github.com/valyala/fasthttp v1.34.0/go.mod h1:epZA5N+7pY6ZaEKRmstzOuYJx9HI8DI1oaCGZpdH4h0= github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= +github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74 h1:JwtAtbp7r/7QSyGz8mKUbYJBg2+6Cd7OjM8o/GNOcVo= github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74/go.mod h1:RmMWU37GKR2s6pgrIEB4ixgpVCt/cf7dnJv3fuH1J1c= github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= diff --git a/integration_test.go b/integration_test.go index d8f7133..413e8e2 100644 --- a/integration_test.go +++ b/integration_test.go @@ -35,10 +35,21 @@ type putResponse struct { OID string `json:"object_id"` } +const ( + testContainerName = "friendly" + versionWithNativeNames = "0.27.5" +) + func TestIntegration(t *testing.T) { rootCtx := context.Background() aioImage := "nspccdev/neofs-aio-testcontainer:" - versions := []string{"0.24.0", "0.25.1", "0.26.1", "0.27.0", "latest"} + versions := []string{ + "0.24.0", + "0.25.1", + "0.26.1", + "0.27.5", + "latest", + } key, err := keys.NewPrivateKeyFromHex("1dd37fba80fec4e6a6f13fd708d8dcb3b29def768017052f6c930fa1c5d90bbb") require.NoError(t, err) @@ -48,13 +59,13 @@ func TestIntegration(t *testing.T) { aioContainer := createDockerContainer(ctx, t, aioImage+version) cancel := runServer() clientPool := getPool(ctx, t, key) - CID, err := createContainer(ctx, t, clientPool) + CID, err := createContainer(ctx, t, clientPool, version) require.NoError(t, err, version) - t.Run("simple put "+version, func(t *testing.T) { simplePut(ctx, t, clientPool, CID) }) - t.Run("simple get "+version, func(t *testing.T) { simpleGet(ctx, t, clientPool, CID) }) - t.Run("get by attribute "+version, func(t *testing.T) { getByAttr(ctx, t, clientPool, CID) }) - t.Run("get zip "+version, func(t *testing.T) { getZip(ctx, t, clientPool, CID) }) + t.Run("simple put "+version, func(t *testing.T) { simplePut(ctx, t, clientPool, CID, version) }) + t.Run("simple get "+version, func(t *testing.T) { simpleGet(ctx, t, clientPool, CID, version) }) + t.Run("get by attribute "+version, func(t *testing.T) { getByAttr(ctx, t, clientPool, CID, version) }) + t.Run("get zip "+version, func(t *testing.T) { getZip(ctx, t, clientPool, CID, version) }) cancel() err = aioContainer.Terminate(ctx) @@ -74,10 +85,19 @@ func runServer() context.CancelFunc { return cancel } -func simplePut(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *cid.ID) { +func simplePut(ctx context.Context, t *testing.T, p *pool.Pool, CID *cid.ID, version string) { + url := "http://localhost:8082/upload/" + CID.String() + makePutRequestAndCheck(ctx, t, p, CID, url) + + if version >= versionWithNativeNames { + url = "http://localhost:8082/upload/" + testContainerName + makePutRequestAndCheck(ctx, t, p, CID, url) + } +} + +func makePutRequestAndCheck(ctx context.Context, t *testing.T, p *pool.Pool, cnrID *cid.ID, url string) { content := "content of file" keyAttr, valAttr := "User-Attribute", "user value" - attributes := map[string]string{ object.AttributeFileName: "newFile.txt", keyAttr: valAttr, @@ -92,23 +112,32 @@ func simplePut(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *ci err = w.Close() require.NoError(t, err) - request, err := http.NewRequest(http.MethodPost, "http://localhost:8082/upload/"+CID.String(), &buff) + request, err := http.NewRequest(http.MethodPost, url, &buff) require.NoError(t, err) request.Header.Set("Content-Type", w.FormDataContentType()) request.Header.Set("X-Attribute-"+keyAttr, valAttr) resp, err := http.DefaultClient.Do(request) require.NoError(t, err) + defer func() { - err = resp.Body.Close() + err := resp.Body.Close() require.NoError(t, err) }() - addr := &putResponse{} - err = json.NewDecoder(resp.Body).Decode(addr) + body, err := io.ReadAll(resp.Body) require.NoError(t, err) - err = CID.DecodeString(addr.CID) + if resp.StatusCode != http.StatusOK { + fmt.Println(string(body)) + } + require.Equal(t, http.StatusOK, resp.StatusCode) + + addr := &putResponse{} + err = json.Unmarshal(body, addr) + require.NoError(t, err) + + err = cnrID.DecodeString(addr.CID) require.NoError(t, err) id := new(oid.ID) @@ -116,7 +145,7 @@ func simplePut(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *ci require.NoError(t, err) objectAddress := address.NewAddress() - objectAddress.SetContainerID(*CID) + objectAddress.SetContainerID(*cnrID) objectAddress.SetObjectID(*id) payload := bytes.NewBuffer(nil) @@ -124,7 +153,7 @@ func simplePut(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *ci var prm pool.PrmObjectGet prm.SetAddress(*objectAddress) - res, err := clientPool.GetObject(ctx, prm) + res, err := p.GetObject(ctx, prm) require.NoError(t, err) _, err = io.Copy(payload, res.Payload) @@ -137,7 +166,7 @@ func simplePut(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *ci } } -func simpleGet(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *cid.ID) { +func simpleGet(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *cid.ID, version string) { content := "content of file" attributes := map[string]string{ "some-attr": "some-get-value", @@ -147,8 +176,18 @@ func simpleGet(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *ci resp, err := http.Get("http://localhost:8082/get/" + CID.String() + "/" + id.String()) require.NoError(t, err) + checkGetResponse(t, resp, content, attributes) + + if version >= versionWithNativeNames { + resp, err = http.Get("http://localhost:8082/get/" + testContainerName + "/" + id.String()) + require.NoError(t, err) + checkGetResponse(t, resp, content, attributes) + } +} + +func checkGetResponse(t *testing.T, resp *http.Response, content string, attributes map[string]string) { defer func() { - err = resp.Body.Close() + err := resp.Body.Close() require.NoError(t, err) }() @@ -161,7 +200,7 @@ func simpleGet(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *ci } } -func getByAttr(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *cid.ID) { +func getByAttr(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *cid.ID, version string) { keyAttr, valAttr := "some-attr", "some-get-by-attr-value" content := "content of file" attributes := map[string]string{keyAttr: valAttr} @@ -176,8 +215,18 @@ func getByAttr(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *ci resp, err := http.Get("http://localhost:8082/get_by_attribute/" + CID.String() + "/" + keyAttr + "/" + valAttr) require.NoError(t, err) + checkGetByAttrResponse(t, resp, content, expectedAttr) + + if version >= versionWithNativeNames { + resp, err = http.Get("http://localhost:8082/get_by_attribute/" + testContainerName + "/" + keyAttr + "/" + valAttr) + require.NoError(t, err) + checkGetByAttrResponse(t, resp, content, expectedAttr) + } +} + +func checkGetByAttrResponse(t *testing.T, resp *http.Response, content string, attributes map[string]string) { defer func() { - err = resp.Body.Close() + err := resp.Body.Close() require.NoError(t, err) }() @@ -185,12 +234,12 @@ func getByAttr(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *ci require.NoError(t, err) require.Equal(t, content, string(data)) - for k, v := range expectedAttr { + for k, v := range attributes { require.Equal(t, v, resp.Header.Get(k)) } } -func getZip(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *cid.ID) { +func getZip(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *cid.ID, version string) { names := []string{"zipfolder/dir/name1.txt", "zipfolder/name2.txt"} contents := []string{"content of file1", "content of file2"} attributes1 := map[string]string{attributeFilePath: names[0]} @@ -199,28 +248,35 @@ func getZip(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *cid.I putObject(ctx, t, clientPool, CID, contents[0], attributes1) putObject(ctx, t, clientPool, CID, contents[1], attributes2) - resp, err := http.Get("http://localhost:8082/zip/" + CID.String() + "/zipfolder") + baseURL := "http://localhost:8082/zip/" + CID.String() + makeZipTest(t, baseURL, names, contents) + + if version >= versionWithNativeNames { + baseURL = "http://localhost:8082/zip/" + testContainerName + makeZipTest(t, baseURL, names, contents) + } +} + +func makeZipTest(t *testing.T, baseURL string, names, contents []string) { + url := baseURL + "/zipfolder" + makeZipRequest(t, url, names, contents) + + // check nested folder + url = baseURL + "/zipfolder/dir" + makeZipRequest(t, url, names[:1], contents[:1]) +} + +func makeZipRequest(t *testing.T, url string, names, contents []string) { + resp, err := http.Get(url) require.NoError(t, err) defer func() { - err = resp.Body.Close() + err := resp.Body.Close() require.NoError(t, err) }() data, err := io.ReadAll(resp.Body) require.NoError(t, err) checkZip(t, data, resp.ContentLength, names, contents) - - // check nested folder - resp2, err := http.Get("http://localhost:8082/zip/" + CID.String() + "/zipfolder/dir") - require.NoError(t, err) - defer func() { - err = resp2.Body.Close() - require.NoError(t, err) - }() - - data2, err := io.ReadAll(resp2.Body) - require.NoError(t, err) - checkZip(t, data2, resp2.ContentLength, names[:1], contents[:1]) } func checkZip(t *testing.T, data []byte, length int64, names, contents []string) { @@ -273,6 +329,8 @@ func getDefaultConfig() *viper.Viper { v.SetDefault(cfgPeers+".0.weight", 1) v.SetDefault(cfgPeers+".0.priority", 1) + v.SetDefault(cfgRPCEndpoint, "http://127.0.0.1:30333") + return v } @@ -290,17 +348,20 @@ func getPool(ctx context.Context, t *testing.T, key *keys.PrivateKey) *pool.Pool return clientPool } -func createContainer(ctx context.Context, t *testing.T, clientPool *pool.Pool) (*cid.ID, error) { +func createContainer(ctx context.Context, t *testing.T, clientPool *pool.Pool, version string) (*cid.ID, error) { pp, err := policy.Parse("REP 1") require.NoError(t, err) cnr := container.New( container.WithPolicy(pp), container.WithCustomBasicACL(0x0FFFFFFF), - container.WithAttribute(container.AttributeName, "friendlyName"), container.WithAttribute(container.AttributeTimestamp, strconv.FormatInt(time.Now().Unix(), 10))) cnr.SetOwnerID(clientPool.OwnerID()) + if version >= versionWithNativeNames { + container.SetNativeName(cnr, testContainerName) + } + var waitPrm pool.WaitParams waitPrm.SetTimeout(15 * time.Second) waitPrm.SetPollInterval(3 * time.Second) diff --git a/resolver/neofs.go b/resolver/neofs.go new file mode 100644 index 0000000..caa460b --- /dev/null +++ b/resolver/neofs.go @@ -0,0 +1,46 @@ +package resolver + +import ( + "context" + "errors" + "fmt" + + "github.com/nspcc-dev/neofs-sdk-go/netmap" + "github.com/nspcc-dev/neofs-sdk-go/pool" +) + +// NeoFSResolver represents virtual connection to the NeoFS network. +// It implements resolver.NeoFS. +type NeoFSResolver struct { + pool *pool.Pool +} + +// NewNeoFSResolver creates new NeoFSResolver using provided pool.Pool. +func NewNeoFSResolver(p *pool.Pool) *NeoFSResolver { + return &NeoFSResolver{pool: p} +} + +// SystemDNS implements resolver.NeoFS interface method. +func (x *NeoFSResolver) SystemDNS(ctx context.Context) (string, error) { + networkInfo, err := x.pool.NetworkInfo(ctx) + if err != nil { + return "", fmt.Errorf("read network info via client: %w", err) + } + + var domain string + + networkInfo.NetworkConfig().IterateParameters(func(parameter *netmap.NetworkParameter) bool { + if string(parameter.Key()) == "SystemDNS" { + domain = string(parameter.Value()) + return true + } + + return false + }) + + if domain == "" { + return "", errors.New("system DNS parameter not found or empty") + } + + return domain, nil +} diff --git a/resolver/resolver.go b/resolver/resolver.go new file mode 100644 index 0000000..9ea9eb8 --- /dev/null +++ b/resolver/resolver.go @@ -0,0 +1,140 @@ +package resolver + +import ( + "context" + "fmt" + + cid "github.com/nspcc-dev/neofs-sdk-go/container/id" + "github.com/nspcc-dev/neofs-sdk-go/ns" +) + +const ( + NNSResolver = "nns" + DNSResolver = "dns" +) + +// NeoFS represents virtual connection to the NeoFS network. +type NeoFS interface { + // SystemDNS reads system DNS network parameters of the NeoFS. + // + // Returns exactly on non-zero value. Returns any error encountered + // which prevented the parameter to be read. + SystemDNS(context.Context) (string, error) +} + +type Config struct { + NeoFS NeoFS + RPCAddress string +} + +type ContainerResolver struct { + Name string + resolve func(context.Context, string) (*cid.ID, error) + + next *ContainerResolver +} + +func (r *ContainerResolver) SetResolveFunc(fn func(context.Context, string) (*cid.ID, error)) { + r.resolve = fn +} + +func (r *ContainerResolver) Resolve(ctx context.Context, name string) (*cid.ID, error) { + cnrID, err := r.resolve(ctx, name) + if err != nil { + if r.next != nil { + cnrID, inErr := r.next.Resolve(ctx, name) + if inErr != nil { + return nil, fmt.Errorf("%s; %w", err.Error(), inErr) + } + return cnrID, nil + } + return nil, err + } + return cnrID, nil +} + +func NewResolver(order []string, cfg *Config) (*ContainerResolver, error) { + if len(order) == 0 { + return nil, fmt.Errorf("resolving order must not be empty") + } + + bucketResolver, err := newResolver(order[len(order)-1], cfg, nil) + if err != nil { + return nil, err + } + + for i := len(order) - 2; i >= 0; i-- { + resolverName := order[i] + next := bucketResolver + + bucketResolver, err = newResolver(resolverName, cfg, next) + if err != nil { + return nil, err + } + } + + return bucketResolver, nil +} + +func newResolver(name string, cfg *Config, next *ContainerResolver) (*ContainerResolver, error) { + switch name { + case DNSResolver: + return NewDNSResolver(cfg.NeoFS, next) + case NNSResolver: + return NewNNSResolver(cfg.RPCAddress, next) + default: + return nil, fmt.Errorf("unknown resolver: %s", name) + } +} + +func NewDNSResolver(neoFS NeoFS, next *ContainerResolver) (*ContainerResolver, error) { + if neoFS == nil { + return nil, fmt.Errorf("pool must not be nil for DNS resolver") + } + + var dns ns.DNS + + resolveFunc := func(ctx context.Context, name string) (*cid.ID, error) { + domain, err := neoFS.SystemDNS(ctx) + if err != nil { + return nil, fmt.Errorf("read system DNS parameter of the NeoFS: %w", err) + } + + domain = name + "." + domain + cnrID, err := dns.ResolveContainerName(domain) + if err != nil { + return nil, fmt.Errorf("couldn't resolve container '%s' as '%s': %w", name, domain, err) + } + return &cnrID, nil + } + + return &ContainerResolver{ + Name: DNSResolver, + + resolve: resolveFunc, + next: next, + }, nil +} + +func NewNNSResolver(rpcAddress string, next *ContainerResolver) (*ContainerResolver, error) { + var nns ns.NNS + + if err := nns.Dial(rpcAddress); err != nil { + return nil, fmt.Errorf("could not dial nns: %w", err) + } + + resolveFunc := func(_ context.Context, name string) (*cid.ID, error) { + cnrID, err := nns.ResolveContainerName(name) + if err != nil { + return nil, fmt.Errorf("couldn't resolve container '%s': %w", name, err) + } + return &cnrID, nil + } + + return &ContainerResolver{ + Name: NNSResolver, + + resolve: resolveFunc, + next: next, + }, nil +} diff --git a/settings.go b/settings.go index 62af367..7a0774f 100644 --- a/settings.go +++ b/settings.go @@ -8,6 +8,7 @@ import ( "strings" "time" + "github.com/nspcc-dev/neofs-http-gw/resolver" "github.com/spf13/pflag" "github.com/spf13/viper" "github.com/valyala/fasthttp" @@ -49,6 +50,12 @@ const ( // Peers. cfgPeers = "peers" + // NeoGo. + cfgRPCEndpoint = "rpc_endpoint" + + // Resolving. + cfgResolveOrder = "resolve_order" + // Zip compression. cfgZipCompression = "zip.compression" @@ -99,6 +106,8 @@ func settings() *viper.Viper { flags.String(cfgTLSKey, "", "TLS key path") peers := flags.StringArrayP(cfgPeers, "p", nil, "NeoFS nodes") + resolveMethods := flags.StringSlice(cfgResolveOrder, []string{resolver.NNSResolver, resolver.DNSResolver}, "set container name resolve order") + // set defaults: // logger: @@ -126,6 +135,10 @@ func settings() *viper.Viper { panic(err) } + if resolveMethods != nil { + v.SetDefault(cfgResolveOrder, *resolveMethods) + } + switch { case help != nil && *help: fmt.Printf("NeoFS HTTP Gateway %s\n", Version) diff --git a/uploader/upload.go b/uploader/upload.go index 3910eac..d60b62f 100644 --- a/uploader/upload.go +++ b/uploader/upload.go @@ -9,11 +9,11 @@ import ( "strconv" "time" + "github.com/nspcc-dev/neofs-http-gw/resolver" "github.com/nspcc-dev/neofs-http-gw/response" "github.com/nspcc-dev/neofs-http-gw/tokens" "github.com/nspcc-dev/neofs-http-gw/utils" "github.com/nspcc-dev/neofs-sdk-go/bearer" - cid "github.com/nspcc-dev/neofs-sdk-go/container/id" "github.com/nspcc-dev/neofs-sdk-go/netmap" "github.com/nspcc-dev/neofs-sdk-go/object" "github.com/nspcc-dev/neofs-sdk-go/object/address" @@ -35,6 +35,7 @@ type Uploader struct { log *zap.Logger pool *pool.Pool enableDefaultTimestamp bool + containerResolver *resolver.ContainerResolver } type epochDurations struct { @@ -45,33 +46,41 @@ type epochDurations struct { // New creates a new Uploader using specified logger, connection pool and // other options. -func New(ctx context.Context, log *zap.Logger, conns *pool.Pool, enableDefaultTimestamp bool) *Uploader { - return &Uploader{ctx, log, conns, enableDefaultTimestamp} +func New(ctx context.Context, params *utils.AppParams, enableDefaultTimestamp bool) *Uploader { + return &Uploader{ + appCtx: ctx, + log: params.Logger, + pool: params.Pool, + enableDefaultTimestamp: enableDefaultTimestamp, + containerResolver: params.Resolver, + } } // Upload handles multipart upload request. func (u *Uploader) Upload(c *fasthttp.RequestCtx) { var ( - err error file MultipartFile idObj *oid.ID addr = address.NewAddress() - idCnr = new(cid.ID) scid, _ = c.UserValue("cid").(string) log = u.log.With(zap.String("cid", scid)) bodyStream = c.RequestBodyStream() drainBuf = make([]byte, drainBufSize) ) - if err = tokens.StoreBearerToken(c); err != nil { + + if err := tokens.StoreBearerToken(c); err != nil { log.Error("could not fetch bearer token", zap.Error(err)) response.Error(c, "could not fetch bearer token", fasthttp.StatusBadRequest) return } - if err = idCnr.DecodeString(scid); err != nil { + + idCnr, err := utils.GetContainerID(u.appCtx, scid, u.containerResolver) + if err != nil { log.Error("wrong container id", zap.Error(err)) response.Error(c, "wrong container id", fasthttp.StatusBadRequest) return } + defer func() { // If the temporary reader can be closed - let's close it. if file == nil { diff --git a/utils/params.go b/utils/params.go new file mode 100644 index 0000000..4bcd352 --- /dev/null +++ b/utils/params.go @@ -0,0 +1,13 @@ +package utils + +import ( + "github.com/nspcc-dev/neofs-http-gw/resolver" + "github.com/nspcc-dev/neofs-sdk-go/pool" + "go.uber.org/zap" +) + +type AppParams struct { + Logger *zap.Logger + Pool *pool.Pool + Resolver *resolver.ContainerResolver +} diff --git a/utils/util.go b/utils/util.go new file mode 100644 index 0000000..73068f1 --- /dev/null +++ b/utils/util.go @@ -0,0 +1,21 @@ +package utils + +import ( + "context" + + "github.com/nspcc-dev/neofs-http-gw/resolver" + cid "github.com/nspcc-dev/neofs-sdk-go/container/id" +) + +// GetContainerID decode container id, if it's not a valid container id +// then trey to resolve name using provided resolver. +func GetContainerID(ctx context.Context, containerID string, resolver *resolver.ContainerResolver) (*cid.ID, error) { + cnrID := new(cid.ID) + err := cnrID.DecodeString(containerID) + if err != nil { + if resolver != nil { + cnrID, err = resolver.Resolve(ctx, containerID) + } + } + return cnrID, err +}