[#142] Support resolving container nicename

Signed-off-by: Denis Kirillov <denis@nspcc.ru>
This commit is contained in:
Denis Kirillov 2022-04-20 12:17:20 +03:00 committed by Kirillov Denis
parent 2b780c1772
commit a42606742a
14 changed files with 458 additions and 74 deletions

View file

@ -189,6 +189,8 @@ and upload objects with it, but deleting, searching, managing ACLs, creating
containers and other activities are not supported and not planned to be
supported.
**Note:** in all download/upload routes you can use container name instead of it's id (`$CID`), but resolvers must be configured properly (see [configs](./config) for examples).
### Preparation
Before uploading or downloading a file make sure you have a prepared container.

58
app.go
View file

@ -13,8 +13,10 @@ import (
"github.com/nspcc-dev/neo-go/pkg/util"
"github.com/nspcc-dev/neo-go/pkg/wallet"
"github.com/nspcc-dev/neofs-http-gw/downloader"
"github.com/nspcc-dev/neofs-http-gw/resolver"
"github.com/nspcc-dev/neofs-http-gw/response"
"github.com/nspcc-dev/neofs-http-gw/uploader"
"github.com/nspcc-dev/neofs-http-gw/utils"
"github.com/nspcc-dev/neofs-sdk-go/pool"
"github.com/spf13/viper"
"github.com/valyala/fasthttp"
@ -28,6 +30,7 @@ type (
cfg *viper.Viper
webServer *fasthttp.Server
webDone chan struct{}
resolver *resolver.ContainerResolver
}
// App is an interface for the main gateway function.
@ -127,9 +130,39 @@ func newApp(ctx context.Context, opt ...Option) App {
if err != nil {
a.log.Fatal("failed to dial pool", zap.Error(err))
}
resolveCfg := &resolver.Config{
NeoFS: resolver.NewNeoFSResolver(a.pool),
RPCAddress: a.cfg.GetString(cfgRPCEndpoint),
}
order := a.cfg.GetStringSlice(cfgResolveOrder)
if resolveCfg.RPCAddress == "" {
order = remove(order, resolver.NNSResolver)
a.log.Warn(fmt.Sprintf("resolver '%s' won't be used since '%s' isn't provided", resolver.NNSResolver, cfgRPCEndpoint))
}
if len(order) != 0 {
a.resolver, err = resolver.NewResolver(order, resolveCfg)
if err != nil {
a.log.Fatal("failed to create resolver", zap.Error(err))
}
} else {
a.log.Info("container resolver is disabled")
}
return a
}
func remove(list []string, element string) []string {
for i, item := range list {
if item == element {
return append(list[:i], list[i+1:]...)
}
}
return list
}
func getNeoFSKey(a *app) (*ecdsa.PrivateKey, error) {
walletPath := a.cfg.GetString(cmdWallet)
if len(walletPath) == 0 {
@ -208,8 +241,9 @@ func (a *app) Serve(ctx context.Context) {
close(a.webDone)
}()
edts := a.cfg.GetBool(cfgUploaderHeaderEnableDefaultTimestamp)
uploader := uploader.New(ctx, a.log, a.pool, edts)
downloader := downloader.New(ctx, a.log, downloader.Settings{ZipCompression: a.cfg.GetBool(cfgZipCompression)}, a.pool)
uploadRoutes := uploader.New(ctx, a.AppParams(), edts)
downloadSettings := downloader.Settings{ZipCompression: a.cfg.GetBool(cfgZipCompression)}
downloadRoutes := downloader.New(ctx, a.AppParams(), downloadSettings)
// Configure router.
r := router.New()
r.RedirectTrailingSlash = true
@ -219,15 +253,15 @@ func (a *app) Serve(ctx context.Context) {
r.MethodNotAllowed = func(r *fasthttp.RequestCtx) {
response.Error(r, "Method Not Allowed", fasthttp.StatusMethodNotAllowed)
}
r.POST("/upload/{cid}", a.logger(uploader.Upload))
r.POST("/upload/{cid}", a.logger(uploadRoutes.Upload))
a.log.Info("added path /upload/{cid}")
r.GET("/get/{cid}/{oid}", a.logger(downloader.DownloadByAddress))
r.HEAD("/get/{cid}/{oid}", a.logger(downloader.HeadByAddress))
r.GET("/get/{cid}/{oid}", a.logger(downloadRoutes.DownloadByAddress))
r.HEAD("/get/{cid}/{oid}", a.logger(downloadRoutes.HeadByAddress))
a.log.Info("added path /get/{cid}/{oid}")
r.GET("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.logger(downloader.DownloadByAttribute))
r.HEAD("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.logger(downloader.HeadByAttribute))
r.GET("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.logger(downloadRoutes.DownloadByAttribute))
r.HEAD("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.logger(downloadRoutes.HeadByAttribute))
a.log.Info("added path /get_by_attribute/{cid}/{attr_key}/{attr_val:*}")
r.GET("/zip/{cid}/{prefix:*}", a.logger(downloader.DownloadZipped))
r.GET("/zip/{cid}/{prefix:*}", a.logger(downloadRoutes.DownloadZipped))
a.log.Info("added path /zip/{cid}/{prefix}")
// enable metrics
if a.cfg.GetBool(cmdMetrics) {
@ -267,3 +301,11 @@ func (a *app) logger(h fasthttp.RequestHandler) fasthttp.RequestHandler {
h(ctx)
})
}
func (a *app) AppParams() *utils.AppParams {
return &utils.AppParams{
Logger: a.log,
Pool: a.pool,
Resolver: a.resolver,
}
}

View file

@ -66,6 +66,11 @@ HTTP_GW_STREAM_REQUEST_BODY=true
# The server rejects requests with bodies exceeding this limit.
HTTP_GW_MAX_REQUEST_BODY_SIZE=4194304
# RPC endpoint to be able to use nns container resolving.
HTTP_GW_RPC_ENDPOINT=http://morph-chain.neofs.devenv:30333
# The order in which resolvers are used to find an container id by name.
HTTP_GW_RESOLVE_ORDER="nns dns"
# Create timestamp for object if it isn't provided by header.
HTTP_GW_UPLOAD_HEADER_USE_DEFAULT_TIMESTAMP=false

View file

@ -66,6 +66,12 @@ web:
# The server rejects requests with bodies exceeding this limit.
max_request_body_size: 4194304
# RPC endpoint to be able to use nns container resolving.
rpc_endpoint: http://morph-chain.neofs.devenv:30333
# The order in which resolvers are used to find an container id by name.
resolve_order:
- nns
- dns
upload_header:
use_default_timestamp: false # Create timestamp for object if it isn't provided by header.

View file

@ -16,6 +16,7 @@ import (
"unicode"
"unicode/utf8"
"github.com/nspcc-dev/neofs-http-gw/resolver"
"github.com/nspcc-dev/neofs-http-gw/response"
"github.com/nspcc-dev/neofs-http-gw/tokens"
"github.com/nspcc-dev/neofs-http-gw/utils"
@ -253,6 +254,7 @@ type Downloader struct {
appCtx context.Context
log *zap.Logger
pool *pool.Pool
containerResolver *resolver.ContainerResolver
settings Settings
}
@ -261,8 +263,14 @@ type Settings struct {
}
// New creates an instance of Downloader using specified options.
func New(ctx context.Context, log *zap.Logger, settings Settings, conns *pool.Pool) *Downloader {
return &Downloader{appCtx: ctx, log: log, pool: conns, settings: settings}
func New(ctx context.Context, params *utils.AppParams, settings Settings) *Downloader {
return &Downloader{
appCtx: ctx,
log: params.Logger,
pool: params.Pool,
settings: settings,
containerResolver: params.Resolver,
}
}
func (d *Downloader) newRequest(ctx *fasthttp.RequestCtx, log *zap.Logger) *request {
@ -282,18 +290,29 @@ func (d *Downloader) DownloadByAddress(c *fasthttp.RequestCtx) {
// prepares request and object address to it.
func (d *Downloader) byAddress(c *fasthttp.RequestCtx, f func(request, *pool.Pool, *address.Address)) {
var (
addr = address.NewAddress()
idCnr, _ = c.UserValue("cid").(string)
idObj, _ = c.UserValue("oid").(string)
val = strings.Join([]string{idCnr, idObj}, "/")
log = d.log.With(zap.String("cid", idCnr), zap.String("oid", idObj))
)
if err := addr.Parse(val); err != nil {
log.Error("wrong object address", zap.Error(err))
response.Error(c, "wrong object address", fasthttp.StatusBadRequest)
cnrID, err := utils.GetContainerID(d.appCtx, idCnr, d.containerResolver)
if err != nil {
log.Error("wrong container id", zap.Error(err))
response.Error(c, "wrong container id", fasthttp.StatusBadRequest)
return
}
objID := new(oid.ID)
if err = objID.DecodeString(idObj); err != nil {
log.Error("wrong object id", zap.Error(err))
response.Error(c, "wrong object id", fasthttp.StatusBadRequest)
return
}
addr := address.NewAddress()
addr.SetContainerID(*cnrID)
addr.SetObjectID(*objID)
f(*d.newRequest(c, log), d.pool, addr)
}
@ -305,16 +324,16 @@ func (d *Downloader) DownloadByAttribute(c *fasthttp.RequestCtx) {
// byAttribute is a wrapper similar to byAddress.
func (d *Downloader) byAttribute(c *fasthttp.RequestCtx, f func(request, *pool.Pool, *address.Address)) {
var (
httpStatus = fasthttp.StatusBadRequest
scid, _ = c.UserValue("cid").(string)
key, _ = url.QueryUnescape(c.UserValue("attr_key").(string))
val, _ = url.QueryUnescape(c.UserValue("attr_val").(string))
log = d.log.With(zap.String("cid", scid), zap.String("attr_key", key), zap.String("attr_val", val))
)
containerID := new(cid.ID)
if err := containerID.DecodeString(scid); err != nil {
containerID, err := utils.GetContainerID(d.appCtx, scid, d.containerResolver)
if err != nil {
log.Error("wrong container id", zap.Error(err))
response.Error(c, "wrong container id", httpStatus)
response.Error(c, "wrong container id", fasthttp.StatusBadRequest)
return
}
@ -370,14 +389,14 @@ func (d *Downloader) DownloadZipped(c *fasthttp.RequestCtx) {
prefix, _ := url.QueryUnescape(c.UserValue("prefix").(string))
log := d.log.With(zap.String("cid", scid), zap.String("prefix", prefix))
containerID := new(cid.ID)
if err := containerID.DecodeString(scid); err != nil {
containerID, err := utils.GetContainerID(d.appCtx, scid, d.containerResolver)
if err != nil {
log.Error("wrong container id", zap.Error(err))
response.Error(c, "wrong container id", fasthttp.StatusBadRequest)
return
}
if err := tokens.StoreBearerToken(c); err != nil {
if err = tokens.StoreBearerToken(c); err != nil {
log.Error("could not fetch and store bearer token", zap.Error(err))
response.Error(c, "could not fetch and store bearer token: "+err.Error(), fasthttp.StatusBadRequest)
return

4
go.mod
View file

@ -42,8 +42,10 @@ require (
github.com/golang/snappy v0.0.4 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/gorilla/mux v1.8.0 // indirect
github.com/gorilla/websocket v1.4.2 // indirect
github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/holiman/uint256 v1.2.0 // indirect
github.com/klauspost/compress v1.15.0 // indirect
github.com/magiconair/properties v1.8.6 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
@ -53,6 +55,7 @@ require (
github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
github.com/morikuni/aec v1.0.0 // indirect
github.com/mr-tron/base58 v1.2.0 // indirect
github.com/nspcc-dev/go-ordered-json v0.0.0-20210915112629-e1b6cce73d02 // indirect
github.com/nspcc-dev/hrw v1.0.9 // indirect
github.com/nspcc-dev/neofs-crypto v0.3.0 // indirect
github.com/nspcc-dev/rfc6979 v0.2.0 // indirect
@ -84,6 +87,7 @@ require (
go.uber.org/multierr v1.7.0 // indirect
golang.org/x/crypto v0.0.0-20220214200702-86341886e292 // indirect
golang.org/x/net v0.0.0-20220412020605-290c469a71a5 // indirect
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad // indirect
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
golang.org/x/text v0.3.7 // indirect

3
go.sum
View file

@ -549,6 +549,7 @@ github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
@ -586,6 +587,7 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO
github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
github.com/holiman/uint256 v1.2.0 h1:gpSYcPLWGv4sG43I2mVLiDZCNDh/EpGjSk8tmtxitHM=
github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
@ -979,6 +981,7 @@ github.com/valyala/fasthttp v1.28.0/go.mod h1:cmWIqlu99AO/RKcp1HWaViTqc57FswJOfY
github.com/valyala/fasthttp v1.34.0 h1:d3AAQJ2DRcxJYHm7OXNXtXt2as1vMDfxeIcFvhmGGm4=
github.com/valyala/fasthttp v1.34.0/go.mod h1:epZA5N+7pY6ZaEKRmstzOuYJx9HI8DI1oaCGZpdH4h0=
github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74 h1:JwtAtbp7r/7QSyGz8mKUbYJBg2+6Cd7OjM8o/GNOcVo=
github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74/go.mod h1:RmMWU37GKR2s6pgrIEB4ixgpVCt/cf7dnJv3fuH1J1c=
github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk=
github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=

View file

@ -35,10 +35,21 @@ type putResponse struct {
OID string `json:"object_id"`
}
const (
testContainerName = "friendly"
versionWithNativeNames = "0.27.5"
)
func TestIntegration(t *testing.T) {
rootCtx := context.Background()
aioImage := "nspccdev/neofs-aio-testcontainer:"
versions := []string{"0.24.0", "0.25.1", "0.26.1", "0.27.0", "latest"}
versions := []string{
"0.24.0",
"0.25.1",
"0.26.1",
"0.27.5",
"latest",
}
key, err := keys.NewPrivateKeyFromHex("1dd37fba80fec4e6a6f13fd708d8dcb3b29def768017052f6c930fa1c5d90bbb")
require.NoError(t, err)
@ -48,13 +59,13 @@ func TestIntegration(t *testing.T) {
aioContainer := createDockerContainer(ctx, t, aioImage+version)
cancel := runServer()
clientPool := getPool(ctx, t, key)
CID, err := createContainer(ctx, t, clientPool)
CID, err := createContainer(ctx, t, clientPool, version)
require.NoError(t, err, version)
t.Run("simple put "+version, func(t *testing.T) { simplePut(ctx, t, clientPool, CID) })
t.Run("simple get "+version, func(t *testing.T) { simpleGet(ctx, t, clientPool, CID) })
t.Run("get by attribute "+version, func(t *testing.T) { getByAttr(ctx, t, clientPool, CID) })
t.Run("get zip "+version, func(t *testing.T) { getZip(ctx, t, clientPool, CID) })
t.Run("simple put "+version, func(t *testing.T) { simplePut(ctx, t, clientPool, CID, version) })
t.Run("simple get "+version, func(t *testing.T) { simpleGet(ctx, t, clientPool, CID, version) })
t.Run("get by attribute "+version, func(t *testing.T) { getByAttr(ctx, t, clientPool, CID, version) })
t.Run("get zip "+version, func(t *testing.T) { getZip(ctx, t, clientPool, CID, version) })
cancel()
err = aioContainer.Terminate(ctx)
@ -74,10 +85,19 @@ func runServer() context.CancelFunc {
return cancel
}
func simplePut(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *cid.ID) {
func simplePut(ctx context.Context, t *testing.T, p *pool.Pool, CID *cid.ID, version string) {
url := "http://localhost:8082/upload/" + CID.String()
makePutRequestAndCheck(ctx, t, p, CID, url)
if version >= versionWithNativeNames {
url = "http://localhost:8082/upload/" + testContainerName
makePutRequestAndCheck(ctx, t, p, CID, url)
}
}
func makePutRequestAndCheck(ctx context.Context, t *testing.T, p *pool.Pool, cnrID *cid.ID, url string) {
content := "content of file"
keyAttr, valAttr := "User-Attribute", "user value"
attributes := map[string]string{
object.AttributeFileName: "newFile.txt",
keyAttr: valAttr,
@ -92,23 +112,32 @@ func simplePut(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *ci
err = w.Close()
require.NoError(t, err)
request, err := http.NewRequest(http.MethodPost, "http://localhost:8082/upload/"+CID.String(), &buff)
request, err := http.NewRequest(http.MethodPost, url, &buff)
require.NoError(t, err)
request.Header.Set("Content-Type", w.FormDataContentType())
request.Header.Set("X-Attribute-"+keyAttr, valAttr)
resp, err := http.DefaultClient.Do(request)
require.NoError(t, err)
defer func() {
err = resp.Body.Close()
err := resp.Body.Close()
require.NoError(t, err)
}()
addr := &putResponse{}
err = json.NewDecoder(resp.Body).Decode(addr)
body, err := io.ReadAll(resp.Body)
require.NoError(t, err)
err = CID.DecodeString(addr.CID)
if resp.StatusCode != http.StatusOK {
fmt.Println(string(body))
}
require.Equal(t, http.StatusOK, resp.StatusCode)
addr := &putResponse{}
err = json.Unmarshal(body, addr)
require.NoError(t, err)
err = cnrID.DecodeString(addr.CID)
require.NoError(t, err)
id := new(oid.ID)
@ -116,7 +145,7 @@ func simplePut(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *ci
require.NoError(t, err)
objectAddress := address.NewAddress()
objectAddress.SetContainerID(*CID)
objectAddress.SetContainerID(*cnrID)
objectAddress.SetObjectID(*id)
payload := bytes.NewBuffer(nil)
@ -124,7 +153,7 @@ func simplePut(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *ci
var prm pool.PrmObjectGet
prm.SetAddress(*objectAddress)
res, err := clientPool.GetObject(ctx, prm)
res, err := p.GetObject(ctx, prm)
require.NoError(t, err)
_, err = io.Copy(payload, res.Payload)
@ -137,7 +166,7 @@ func simplePut(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *ci
}
}
func simpleGet(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *cid.ID) {
func simpleGet(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *cid.ID, version string) {
content := "content of file"
attributes := map[string]string{
"some-attr": "some-get-value",
@ -147,8 +176,18 @@ func simpleGet(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *ci
resp, err := http.Get("http://localhost:8082/get/" + CID.String() + "/" + id.String())
require.NoError(t, err)
checkGetResponse(t, resp, content, attributes)
if version >= versionWithNativeNames {
resp, err = http.Get("http://localhost:8082/get/" + testContainerName + "/" + id.String())
require.NoError(t, err)
checkGetResponse(t, resp, content, attributes)
}
}
func checkGetResponse(t *testing.T, resp *http.Response, content string, attributes map[string]string) {
defer func() {
err = resp.Body.Close()
err := resp.Body.Close()
require.NoError(t, err)
}()
@ -161,7 +200,7 @@ func simpleGet(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *ci
}
}
func getByAttr(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *cid.ID) {
func getByAttr(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *cid.ID, version string) {
keyAttr, valAttr := "some-attr", "some-get-by-attr-value"
content := "content of file"
attributes := map[string]string{keyAttr: valAttr}
@ -176,8 +215,18 @@ func getByAttr(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *ci
resp, err := http.Get("http://localhost:8082/get_by_attribute/" + CID.String() + "/" + keyAttr + "/" + valAttr)
require.NoError(t, err)
checkGetByAttrResponse(t, resp, content, expectedAttr)
if version >= versionWithNativeNames {
resp, err = http.Get("http://localhost:8082/get_by_attribute/" + testContainerName + "/" + keyAttr + "/" + valAttr)
require.NoError(t, err)
checkGetByAttrResponse(t, resp, content, expectedAttr)
}
}
func checkGetByAttrResponse(t *testing.T, resp *http.Response, content string, attributes map[string]string) {
defer func() {
err = resp.Body.Close()
err := resp.Body.Close()
require.NoError(t, err)
}()
@ -185,12 +234,12 @@ func getByAttr(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *ci
require.NoError(t, err)
require.Equal(t, content, string(data))
for k, v := range expectedAttr {
for k, v := range attributes {
require.Equal(t, v, resp.Header.Get(k))
}
}
func getZip(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *cid.ID) {
func getZip(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *cid.ID, version string) {
names := []string{"zipfolder/dir/name1.txt", "zipfolder/name2.txt"}
contents := []string{"content of file1", "content of file2"}
attributes1 := map[string]string{attributeFilePath: names[0]}
@ -199,28 +248,35 @@ func getZip(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *cid.I
putObject(ctx, t, clientPool, CID, contents[0], attributes1)
putObject(ctx, t, clientPool, CID, contents[1], attributes2)
resp, err := http.Get("http://localhost:8082/zip/" + CID.String() + "/zipfolder")
baseURL := "http://localhost:8082/zip/" + CID.String()
makeZipTest(t, baseURL, names, contents)
if version >= versionWithNativeNames {
baseURL = "http://localhost:8082/zip/" + testContainerName
makeZipTest(t, baseURL, names, contents)
}
}
func makeZipTest(t *testing.T, baseURL string, names, contents []string) {
url := baseURL + "/zipfolder"
makeZipRequest(t, url, names, contents)
// check nested folder
url = baseURL + "/zipfolder/dir"
makeZipRequest(t, url, names[:1], contents[:1])
}
func makeZipRequest(t *testing.T, url string, names, contents []string) {
resp, err := http.Get(url)
require.NoError(t, err)
defer func() {
err = resp.Body.Close()
err := resp.Body.Close()
require.NoError(t, err)
}()
data, err := io.ReadAll(resp.Body)
require.NoError(t, err)
checkZip(t, data, resp.ContentLength, names, contents)
// check nested folder
resp2, err := http.Get("http://localhost:8082/zip/" + CID.String() + "/zipfolder/dir")
require.NoError(t, err)
defer func() {
err = resp2.Body.Close()
require.NoError(t, err)
}()
data2, err := io.ReadAll(resp2.Body)
require.NoError(t, err)
checkZip(t, data2, resp2.ContentLength, names[:1], contents[:1])
}
func checkZip(t *testing.T, data []byte, length int64, names, contents []string) {
@ -273,6 +329,8 @@ func getDefaultConfig() *viper.Viper {
v.SetDefault(cfgPeers+".0.weight", 1)
v.SetDefault(cfgPeers+".0.priority", 1)
v.SetDefault(cfgRPCEndpoint, "http://127.0.0.1:30333")
return v
}
@ -290,17 +348,20 @@ func getPool(ctx context.Context, t *testing.T, key *keys.PrivateKey) *pool.Pool
return clientPool
}
func createContainer(ctx context.Context, t *testing.T, clientPool *pool.Pool) (*cid.ID, error) {
func createContainer(ctx context.Context, t *testing.T, clientPool *pool.Pool, version string) (*cid.ID, error) {
pp, err := policy.Parse("REP 1")
require.NoError(t, err)
cnr := container.New(
container.WithPolicy(pp),
container.WithCustomBasicACL(0x0FFFFFFF),
container.WithAttribute(container.AttributeName, "friendlyName"),
container.WithAttribute(container.AttributeTimestamp, strconv.FormatInt(time.Now().Unix(), 10)))
cnr.SetOwnerID(clientPool.OwnerID())
if version >= versionWithNativeNames {
container.SetNativeName(cnr, testContainerName)
}
var waitPrm pool.WaitParams
waitPrm.SetTimeout(15 * time.Second)
waitPrm.SetPollInterval(3 * time.Second)

46
resolver/neofs.go Normal file
View file

@ -0,0 +1,46 @@
package resolver
import (
"context"
"errors"
"fmt"
"github.com/nspcc-dev/neofs-sdk-go/netmap"
"github.com/nspcc-dev/neofs-sdk-go/pool"
)
// NeoFSResolver represents virtual connection to the NeoFS network.
// It implements resolver.NeoFS.
type NeoFSResolver struct {
pool *pool.Pool
}
// NewNeoFSResolver creates new NeoFSResolver using provided pool.Pool.
func NewNeoFSResolver(p *pool.Pool) *NeoFSResolver {
return &NeoFSResolver{pool: p}
}
// SystemDNS implements resolver.NeoFS interface method.
func (x *NeoFSResolver) SystemDNS(ctx context.Context) (string, error) {
networkInfo, err := x.pool.NetworkInfo(ctx)
if err != nil {
return "", fmt.Errorf("read network info via client: %w", err)
}
var domain string
networkInfo.NetworkConfig().IterateParameters(func(parameter *netmap.NetworkParameter) bool {
if string(parameter.Key()) == "SystemDNS" {
domain = string(parameter.Value())
return true
}
return false
})
if domain == "" {
return "", errors.New("system DNS parameter not found or empty")
}
return domain, nil
}

140
resolver/resolver.go Normal file
View file

@ -0,0 +1,140 @@
package resolver
import (
"context"
"fmt"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
"github.com/nspcc-dev/neofs-sdk-go/ns"
)
const (
NNSResolver = "nns"
DNSResolver = "dns"
)
// NeoFS represents virtual connection to the NeoFS network.
type NeoFS interface {
// SystemDNS reads system DNS network parameters of the NeoFS.
//
// Returns exactly on non-zero value. Returns any error encountered
// which prevented the parameter to be read.
SystemDNS(context.Context) (string, error)
}
type Config struct {
NeoFS NeoFS
RPCAddress string
}
type ContainerResolver struct {
Name string
resolve func(context.Context, string) (*cid.ID, error)
next *ContainerResolver
}
func (r *ContainerResolver) SetResolveFunc(fn func(context.Context, string) (*cid.ID, error)) {
r.resolve = fn
}
func (r *ContainerResolver) Resolve(ctx context.Context, name string) (*cid.ID, error) {
cnrID, err := r.resolve(ctx, name)
if err != nil {
if r.next != nil {
cnrID, inErr := r.next.Resolve(ctx, name)
if inErr != nil {
return nil, fmt.Errorf("%s; %w", err.Error(), inErr)
}
return cnrID, nil
}
return nil, err
}
return cnrID, nil
}
func NewResolver(order []string, cfg *Config) (*ContainerResolver, error) {
if len(order) == 0 {
return nil, fmt.Errorf("resolving order must not be empty")
}
bucketResolver, err := newResolver(order[len(order)-1], cfg, nil)
if err != nil {
return nil, err
}
for i := len(order) - 2; i >= 0; i-- {
resolverName := order[i]
next := bucketResolver
bucketResolver, err = newResolver(resolverName, cfg, next)
if err != nil {
return nil, err
}
}
return bucketResolver, nil
}
func newResolver(name string, cfg *Config, next *ContainerResolver) (*ContainerResolver, error) {
switch name {
case DNSResolver:
return NewDNSResolver(cfg.NeoFS, next)
case NNSResolver:
return NewNNSResolver(cfg.RPCAddress, next)
default:
return nil, fmt.Errorf("unknown resolver: %s", name)
}
}
func NewDNSResolver(neoFS NeoFS, next *ContainerResolver) (*ContainerResolver, error) {
if neoFS == nil {
return nil, fmt.Errorf("pool must not be nil for DNS resolver")
}
var dns ns.DNS
resolveFunc := func(ctx context.Context, name string) (*cid.ID, error) {
domain, err := neoFS.SystemDNS(ctx)
if err != nil {
return nil, fmt.Errorf("read system DNS parameter of the NeoFS: %w", err)
}
domain = name + "." + domain
cnrID, err := dns.ResolveContainerName(domain)
if err != nil {
return nil, fmt.Errorf("couldn't resolve container '%s' as '%s': %w", name, domain, err)
}
return &cnrID, nil
}
return &ContainerResolver{
Name: DNSResolver,
resolve: resolveFunc,
next: next,
}, nil
}
func NewNNSResolver(rpcAddress string, next *ContainerResolver) (*ContainerResolver, error) {
var nns ns.NNS
if err := nns.Dial(rpcAddress); err != nil {
return nil, fmt.Errorf("could not dial nns: %w", err)
}
resolveFunc := func(_ context.Context, name string) (*cid.ID, error) {
cnrID, err := nns.ResolveContainerName(name)
if err != nil {
return nil, fmt.Errorf("couldn't resolve container '%s': %w", name, err)
}
return &cnrID, nil
}
return &ContainerResolver{
Name: NNSResolver,
resolve: resolveFunc,
next: next,
}, nil
}

View file

@ -8,6 +8,7 @@ import (
"strings"
"time"
"github.com/nspcc-dev/neofs-http-gw/resolver"
"github.com/spf13/pflag"
"github.com/spf13/viper"
"github.com/valyala/fasthttp"
@ -49,6 +50,12 @@ const (
// Peers.
cfgPeers = "peers"
// NeoGo.
cfgRPCEndpoint = "rpc_endpoint"
// Resolving.
cfgResolveOrder = "resolve_order"
// Zip compression.
cfgZipCompression = "zip.compression"
@ -99,6 +106,8 @@ func settings() *viper.Viper {
flags.String(cfgTLSKey, "", "TLS key path")
peers := flags.StringArrayP(cfgPeers, "p", nil, "NeoFS nodes")
resolveMethods := flags.StringSlice(cfgResolveOrder, []string{resolver.NNSResolver, resolver.DNSResolver}, "set container name resolve order")
// set defaults:
// logger:
@ -126,6 +135,10 @@ func settings() *viper.Viper {
panic(err)
}
if resolveMethods != nil {
v.SetDefault(cfgResolveOrder, *resolveMethods)
}
switch {
case help != nil && *help:
fmt.Printf("NeoFS HTTP Gateway %s\n", Version)

View file

@ -9,11 +9,11 @@ import (
"strconv"
"time"
"github.com/nspcc-dev/neofs-http-gw/resolver"
"github.com/nspcc-dev/neofs-http-gw/response"
"github.com/nspcc-dev/neofs-http-gw/tokens"
"github.com/nspcc-dev/neofs-http-gw/utils"
"github.com/nspcc-dev/neofs-sdk-go/bearer"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
"github.com/nspcc-dev/neofs-sdk-go/netmap"
"github.com/nspcc-dev/neofs-sdk-go/object"
"github.com/nspcc-dev/neofs-sdk-go/object/address"
@ -35,6 +35,7 @@ type Uploader struct {
log *zap.Logger
pool *pool.Pool
enableDefaultTimestamp bool
containerResolver *resolver.ContainerResolver
}
type epochDurations struct {
@ -45,33 +46,41 @@ type epochDurations struct {
// New creates a new Uploader using specified logger, connection pool and
// other options.
func New(ctx context.Context, log *zap.Logger, conns *pool.Pool, enableDefaultTimestamp bool) *Uploader {
return &Uploader{ctx, log, conns, enableDefaultTimestamp}
func New(ctx context.Context, params *utils.AppParams, enableDefaultTimestamp bool) *Uploader {
return &Uploader{
appCtx: ctx,
log: params.Logger,
pool: params.Pool,
enableDefaultTimestamp: enableDefaultTimestamp,
containerResolver: params.Resolver,
}
}
// Upload handles multipart upload request.
func (u *Uploader) Upload(c *fasthttp.RequestCtx) {
var (
err error
file MultipartFile
idObj *oid.ID
addr = address.NewAddress()
idCnr = new(cid.ID)
scid, _ = c.UserValue("cid").(string)
log = u.log.With(zap.String("cid", scid))
bodyStream = c.RequestBodyStream()
drainBuf = make([]byte, drainBufSize)
)
if err = tokens.StoreBearerToken(c); err != nil {
if err := tokens.StoreBearerToken(c); err != nil {
log.Error("could not fetch bearer token", zap.Error(err))
response.Error(c, "could not fetch bearer token", fasthttp.StatusBadRequest)
return
}
if err = idCnr.DecodeString(scid); err != nil {
idCnr, err := utils.GetContainerID(u.appCtx, scid, u.containerResolver)
if err != nil {
log.Error("wrong container id", zap.Error(err))
response.Error(c, "wrong container id", fasthttp.StatusBadRequest)
return
}
defer func() {
// If the temporary reader can be closed - let's close it.
if file == nil {

13
utils/params.go Normal file
View file

@ -0,0 +1,13 @@
package utils
import (
"github.com/nspcc-dev/neofs-http-gw/resolver"
"github.com/nspcc-dev/neofs-sdk-go/pool"
"go.uber.org/zap"
)
type AppParams struct {
Logger *zap.Logger
Pool *pool.Pool
Resolver *resolver.ContainerResolver
}

21
utils/util.go Normal file
View file

@ -0,0 +1,21 @@
package utils
import (
"context"
"github.com/nspcc-dev/neofs-http-gw/resolver"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
)
// GetContainerID decode container id, if it's not a valid container id
// then trey to resolve name using provided resolver.
func GetContainerID(ctx context.Context, containerID string, resolver *resolver.ContainerResolver) (*cid.ID, error) {
cnrID := new(cid.ID)
err := cnrID.DecodeString(containerID)
if err != nil {
if resolver != nil {
cnrID, err = resolver.Resolve(ctx, containerID)
}
}
return cnrID, err
}