[#142] Support resolving container nicename
Signed-off-by: Denis Kirillov <denis@nspcc.ru>
This commit is contained in:
parent
2b780c1772
commit
a42606742a
14 changed files with 458 additions and 74 deletions
|
@ -189,6 +189,8 @@ and upload objects with it, but deleting, searching, managing ACLs, creating
|
|||
containers and other activities are not supported and not planned to be
|
||||
supported.
|
||||
|
||||
**Note:** in all download/upload routes you can use container name instead of it's id (`$CID`), but resolvers must be configured properly (see [configs](./config) for examples).
|
||||
|
||||
### Preparation
|
||||
|
||||
Before uploading or downloading a file make sure you have a prepared container.
|
||||
|
|
58
app.go
58
app.go
|
@ -13,8 +13,10 @@ import (
|
|||
"github.com/nspcc-dev/neo-go/pkg/util"
|
||||
"github.com/nspcc-dev/neo-go/pkg/wallet"
|
||||
"github.com/nspcc-dev/neofs-http-gw/downloader"
|
||||
"github.com/nspcc-dev/neofs-http-gw/resolver"
|
||||
"github.com/nspcc-dev/neofs-http-gw/response"
|
||||
"github.com/nspcc-dev/neofs-http-gw/uploader"
|
||||
"github.com/nspcc-dev/neofs-http-gw/utils"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/pool"
|
||||
"github.com/spf13/viper"
|
||||
"github.com/valyala/fasthttp"
|
||||
|
@ -28,6 +30,7 @@ type (
|
|||
cfg *viper.Viper
|
||||
webServer *fasthttp.Server
|
||||
webDone chan struct{}
|
||||
resolver *resolver.ContainerResolver
|
||||
}
|
||||
|
||||
// App is an interface for the main gateway function.
|
||||
|
@ -127,9 +130,39 @@ func newApp(ctx context.Context, opt ...Option) App {
|
|||
if err != nil {
|
||||
a.log.Fatal("failed to dial pool", zap.Error(err))
|
||||
}
|
||||
|
||||
resolveCfg := &resolver.Config{
|
||||
NeoFS: resolver.NewNeoFSResolver(a.pool),
|
||||
RPCAddress: a.cfg.GetString(cfgRPCEndpoint),
|
||||
}
|
||||
|
||||
order := a.cfg.GetStringSlice(cfgResolveOrder)
|
||||
if resolveCfg.RPCAddress == "" {
|
||||
order = remove(order, resolver.NNSResolver)
|
||||
a.log.Warn(fmt.Sprintf("resolver '%s' won't be used since '%s' isn't provided", resolver.NNSResolver, cfgRPCEndpoint))
|
||||
}
|
||||
|
||||
if len(order) != 0 {
|
||||
a.resolver, err = resolver.NewResolver(order, resolveCfg)
|
||||
if err != nil {
|
||||
a.log.Fatal("failed to create resolver", zap.Error(err))
|
||||
}
|
||||
} else {
|
||||
a.log.Info("container resolver is disabled")
|
||||
}
|
||||
|
||||
return a
|
||||
}
|
||||
|
||||
func remove(list []string, element string) []string {
|
||||
for i, item := range list {
|
||||
if item == element {
|
||||
return append(list[:i], list[i+1:]...)
|
||||
}
|
||||
}
|
||||
return list
|
||||
}
|
||||
|
||||
func getNeoFSKey(a *app) (*ecdsa.PrivateKey, error) {
|
||||
walletPath := a.cfg.GetString(cmdWallet)
|
||||
if len(walletPath) == 0 {
|
||||
|
@ -208,8 +241,9 @@ func (a *app) Serve(ctx context.Context) {
|
|||
close(a.webDone)
|
||||
}()
|
||||
edts := a.cfg.GetBool(cfgUploaderHeaderEnableDefaultTimestamp)
|
||||
uploader := uploader.New(ctx, a.log, a.pool, edts)
|
||||
downloader := downloader.New(ctx, a.log, downloader.Settings{ZipCompression: a.cfg.GetBool(cfgZipCompression)}, a.pool)
|
||||
uploadRoutes := uploader.New(ctx, a.AppParams(), edts)
|
||||
downloadSettings := downloader.Settings{ZipCompression: a.cfg.GetBool(cfgZipCompression)}
|
||||
downloadRoutes := downloader.New(ctx, a.AppParams(), downloadSettings)
|
||||
// Configure router.
|
||||
r := router.New()
|
||||
r.RedirectTrailingSlash = true
|
||||
|
@ -219,15 +253,15 @@ func (a *app) Serve(ctx context.Context) {
|
|||
r.MethodNotAllowed = func(r *fasthttp.RequestCtx) {
|
||||
response.Error(r, "Method Not Allowed", fasthttp.StatusMethodNotAllowed)
|
||||
}
|
||||
r.POST("/upload/{cid}", a.logger(uploader.Upload))
|
||||
r.POST("/upload/{cid}", a.logger(uploadRoutes.Upload))
|
||||
a.log.Info("added path /upload/{cid}")
|
||||
r.GET("/get/{cid}/{oid}", a.logger(downloader.DownloadByAddress))
|
||||
r.HEAD("/get/{cid}/{oid}", a.logger(downloader.HeadByAddress))
|
||||
r.GET("/get/{cid}/{oid}", a.logger(downloadRoutes.DownloadByAddress))
|
||||
r.HEAD("/get/{cid}/{oid}", a.logger(downloadRoutes.HeadByAddress))
|
||||
a.log.Info("added path /get/{cid}/{oid}")
|
||||
r.GET("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.logger(downloader.DownloadByAttribute))
|
||||
r.HEAD("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.logger(downloader.HeadByAttribute))
|
||||
r.GET("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.logger(downloadRoutes.DownloadByAttribute))
|
||||
r.HEAD("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.logger(downloadRoutes.HeadByAttribute))
|
||||
a.log.Info("added path /get_by_attribute/{cid}/{attr_key}/{attr_val:*}")
|
||||
r.GET("/zip/{cid}/{prefix:*}", a.logger(downloader.DownloadZipped))
|
||||
r.GET("/zip/{cid}/{prefix:*}", a.logger(downloadRoutes.DownloadZipped))
|
||||
a.log.Info("added path /zip/{cid}/{prefix}")
|
||||
// enable metrics
|
||||
if a.cfg.GetBool(cmdMetrics) {
|
||||
|
@ -267,3 +301,11 @@ func (a *app) logger(h fasthttp.RequestHandler) fasthttp.RequestHandler {
|
|||
h(ctx)
|
||||
})
|
||||
}
|
||||
|
||||
func (a *app) AppParams() *utils.AppParams {
|
||||
return &utils.AppParams{
|
||||
Logger: a.log,
|
||||
Pool: a.pool,
|
||||
Resolver: a.resolver,
|
||||
}
|
||||
}
|
||||
|
|
|
@ -66,6 +66,11 @@ HTTP_GW_STREAM_REQUEST_BODY=true
|
|||
# The server rejects requests with bodies exceeding this limit.
|
||||
HTTP_GW_MAX_REQUEST_BODY_SIZE=4194304
|
||||
|
||||
# RPC endpoint to be able to use nns container resolving.
|
||||
HTTP_GW_RPC_ENDPOINT=http://morph-chain.neofs.devenv:30333
|
||||
# The order in which resolvers are used to find an container id by name.
|
||||
HTTP_GW_RESOLVE_ORDER="nns dns"
|
||||
|
||||
# Create timestamp for object if it isn't provided by header.
|
||||
HTTP_GW_UPLOAD_HEADER_USE_DEFAULT_TIMESTAMP=false
|
||||
|
||||
|
|
|
@ -66,6 +66,12 @@ web:
|
|||
# The server rejects requests with bodies exceeding this limit.
|
||||
max_request_body_size: 4194304
|
||||
|
||||
# RPC endpoint to be able to use nns container resolving.
|
||||
rpc_endpoint: http://morph-chain.neofs.devenv:30333
|
||||
# The order in which resolvers are used to find an container id by name.
|
||||
resolve_order:
|
||||
- nns
|
||||
- dns
|
||||
|
||||
upload_header:
|
||||
use_default_timestamp: false # Create timestamp for object if it isn't provided by header.
|
||||
|
|
|
@ -16,6 +16,7 @@ import (
|
|||
"unicode"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/nspcc-dev/neofs-http-gw/resolver"
|
||||
"github.com/nspcc-dev/neofs-http-gw/response"
|
||||
"github.com/nspcc-dev/neofs-http-gw/tokens"
|
||||
"github.com/nspcc-dev/neofs-http-gw/utils"
|
||||
|
@ -253,6 +254,7 @@ type Downloader struct {
|
|||
appCtx context.Context
|
||||
log *zap.Logger
|
||||
pool *pool.Pool
|
||||
containerResolver *resolver.ContainerResolver
|
||||
settings Settings
|
||||
}
|
||||
|
||||
|
@ -261,8 +263,14 @@ type Settings struct {
|
|||
}
|
||||
|
||||
// New creates an instance of Downloader using specified options.
|
||||
func New(ctx context.Context, log *zap.Logger, settings Settings, conns *pool.Pool) *Downloader {
|
||||
return &Downloader{appCtx: ctx, log: log, pool: conns, settings: settings}
|
||||
func New(ctx context.Context, params *utils.AppParams, settings Settings) *Downloader {
|
||||
return &Downloader{
|
||||
appCtx: ctx,
|
||||
log: params.Logger,
|
||||
pool: params.Pool,
|
||||
settings: settings,
|
||||
containerResolver: params.Resolver,
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Downloader) newRequest(ctx *fasthttp.RequestCtx, log *zap.Logger) *request {
|
||||
|
@ -282,18 +290,29 @@ func (d *Downloader) DownloadByAddress(c *fasthttp.RequestCtx) {
|
|||
// prepares request and object address to it.
|
||||
func (d *Downloader) byAddress(c *fasthttp.RequestCtx, f func(request, *pool.Pool, *address.Address)) {
|
||||
var (
|
||||
addr = address.NewAddress()
|
||||
idCnr, _ = c.UserValue("cid").(string)
|
||||
idObj, _ = c.UserValue("oid").(string)
|
||||
val = strings.Join([]string{idCnr, idObj}, "/")
|
||||
log = d.log.With(zap.String("cid", idCnr), zap.String("oid", idObj))
|
||||
)
|
||||
if err := addr.Parse(val); err != nil {
|
||||
log.Error("wrong object address", zap.Error(err))
|
||||
response.Error(c, "wrong object address", fasthttp.StatusBadRequest)
|
||||
|
||||
cnrID, err := utils.GetContainerID(d.appCtx, idCnr, d.containerResolver)
|
||||
if err != nil {
|
||||
log.Error("wrong container id", zap.Error(err))
|
||||
response.Error(c, "wrong container id", fasthttp.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
objID := new(oid.ID)
|
||||
if err = objID.DecodeString(idObj); err != nil {
|
||||
log.Error("wrong object id", zap.Error(err))
|
||||
response.Error(c, "wrong object id", fasthttp.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
addr := address.NewAddress()
|
||||
addr.SetContainerID(*cnrID)
|
||||
addr.SetObjectID(*objID)
|
||||
|
||||
f(*d.newRequest(c, log), d.pool, addr)
|
||||
}
|
||||
|
||||
|
@ -305,16 +324,16 @@ func (d *Downloader) DownloadByAttribute(c *fasthttp.RequestCtx) {
|
|||
// byAttribute is a wrapper similar to byAddress.
|
||||
func (d *Downloader) byAttribute(c *fasthttp.RequestCtx, f func(request, *pool.Pool, *address.Address)) {
|
||||
var (
|
||||
httpStatus = fasthttp.StatusBadRequest
|
||||
scid, _ = c.UserValue("cid").(string)
|
||||
key, _ = url.QueryUnescape(c.UserValue("attr_key").(string))
|
||||
val, _ = url.QueryUnescape(c.UserValue("attr_val").(string))
|
||||
log = d.log.With(zap.String("cid", scid), zap.String("attr_key", key), zap.String("attr_val", val))
|
||||
)
|
||||
containerID := new(cid.ID)
|
||||
if err := containerID.DecodeString(scid); err != nil {
|
||||
|
||||
containerID, err := utils.GetContainerID(d.appCtx, scid, d.containerResolver)
|
||||
if err != nil {
|
||||
log.Error("wrong container id", zap.Error(err))
|
||||
response.Error(c, "wrong container id", httpStatus)
|
||||
response.Error(c, "wrong container id", fasthttp.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
|
@ -370,14 +389,14 @@ func (d *Downloader) DownloadZipped(c *fasthttp.RequestCtx) {
|
|||
prefix, _ := url.QueryUnescape(c.UserValue("prefix").(string))
|
||||
log := d.log.With(zap.String("cid", scid), zap.String("prefix", prefix))
|
||||
|
||||
containerID := new(cid.ID)
|
||||
if err := containerID.DecodeString(scid); err != nil {
|
||||
containerID, err := utils.GetContainerID(d.appCtx, scid, d.containerResolver)
|
||||
if err != nil {
|
||||
log.Error("wrong container id", zap.Error(err))
|
||||
response.Error(c, "wrong container id", fasthttp.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
if err := tokens.StoreBearerToken(c); err != nil {
|
||||
if err = tokens.StoreBearerToken(c); err != nil {
|
||||
log.Error("could not fetch and store bearer token", zap.Error(err))
|
||||
response.Error(c, "could not fetch and store bearer token: "+err.Error(), fasthttp.StatusBadRequest)
|
||||
return
|
||||
|
|
4
go.mod
4
go.mod
|
@ -42,8 +42,10 @@ require (
|
|||
github.com/golang/snappy v0.0.4 // indirect
|
||||
github.com/google/uuid v1.3.0 // indirect
|
||||
github.com/gorilla/mux v1.8.0 // indirect
|
||||
github.com/gorilla/websocket v1.4.2 // indirect
|
||||
github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect
|
||||
github.com/hashicorp/hcl v1.0.0 // indirect
|
||||
github.com/holiman/uint256 v1.2.0 // indirect
|
||||
github.com/klauspost/compress v1.15.0 // indirect
|
||||
github.com/magiconair/properties v1.8.6 // indirect
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
|
||||
|
@ -53,6 +55,7 @@ require (
|
|||
github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
|
||||
github.com/morikuni/aec v1.0.0 // indirect
|
||||
github.com/mr-tron/base58 v1.2.0 // indirect
|
||||
github.com/nspcc-dev/go-ordered-json v0.0.0-20210915112629-e1b6cce73d02 // indirect
|
||||
github.com/nspcc-dev/hrw v1.0.9 // indirect
|
||||
github.com/nspcc-dev/neofs-crypto v0.3.0 // indirect
|
||||
github.com/nspcc-dev/rfc6979 v0.2.0 // indirect
|
||||
|
@ -84,6 +87,7 @@ require (
|
|||
go.uber.org/multierr v1.7.0 // indirect
|
||||
golang.org/x/crypto v0.0.0-20220214200702-86341886e292 // indirect
|
||||
golang.org/x/net v0.0.0-20220412020605-290c469a71a5 // indirect
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
|
||||
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad // indirect
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
|
||||
golang.org/x/text v0.3.7 // indirect
|
||||
|
|
3
go.sum
3
go.sum
|
@ -549,6 +549,7 @@ github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
|
|||
github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
|
||||
github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
|
||||
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
|
||||
github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
|
||||
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
|
||||
github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
|
||||
|
@ -586,6 +587,7 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO
|
|||
github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
|
||||
github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
|
||||
github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
|
||||
github.com/holiman/uint256 v1.2.0 h1:gpSYcPLWGv4sG43I2mVLiDZCNDh/EpGjSk8tmtxitHM=
|
||||
github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw=
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
|
||||
|
@ -979,6 +981,7 @@ github.com/valyala/fasthttp v1.28.0/go.mod h1:cmWIqlu99AO/RKcp1HWaViTqc57FswJOfY
|
|||
github.com/valyala/fasthttp v1.34.0 h1:d3AAQJ2DRcxJYHm7OXNXtXt2as1vMDfxeIcFvhmGGm4=
|
||||
github.com/valyala/fasthttp v1.34.0/go.mod h1:epZA5N+7pY6ZaEKRmstzOuYJx9HI8DI1oaCGZpdH4h0=
|
||||
github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
|
||||
github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74 h1:JwtAtbp7r/7QSyGz8mKUbYJBg2+6Cd7OjM8o/GNOcVo=
|
||||
github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74/go.mod h1:RmMWU37GKR2s6pgrIEB4ixgpVCt/cf7dnJv3fuH1J1c=
|
||||
github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk=
|
||||
github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
|
||||
|
|
|
@ -35,10 +35,21 @@ type putResponse struct {
|
|||
OID string `json:"object_id"`
|
||||
}
|
||||
|
||||
const (
|
||||
testContainerName = "friendly"
|
||||
versionWithNativeNames = "0.27.5"
|
||||
)
|
||||
|
||||
func TestIntegration(t *testing.T) {
|
||||
rootCtx := context.Background()
|
||||
aioImage := "nspccdev/neofs-aio-testcontainer:"
|
||||
versions := []string{"0.24.0", "0.25.1", "0.26.1", "0.27.0", "latest"}
|
||||
versions := []string{
|
||||
"0.24.0",
|
||||
"0.25.1",
|
||||
"0.26.1",
|
||||
"0.27.5",
|
||||
"latest",
|
||||
}
|
||||
key, err := keys.NewPrivateKeyFromHex("1dd37fba80fec4e6a6f13fd708d8dcb3b29def768017052f6c930fa1c5d90bbb")
|
||||
require.NoError(t, err)
|
||||
|
||||
|
@ -48,13 +59,13 @@ func TestIntegration(t *testing.T) {
|
|||
aioContainer := createDockerContainer(ctx, t, aioImage+version)
|
||||
cancel := runServer()
|
||||
clientPool := getPool(ctx, t, key)
|
||||
CID, err := createContainer(ctx, t, clientPool)
|
||||
CID, err := createContainer(ctx, t, clientPool, version)
|
||||
require.NoError(t, err, version)
|
||||
|
||||
t.Run("simple put "+version, func(t *testing.T) { simplePut(ctx, t, clientPool, CID) })
|
||||
t.Run("simple get "+version, func(t *testing.T) { simpleGet(ctx, t, clientPool, CID) })
|
||||
t.Run("get by attribute "+version, func(t *testing.T) { getByAttr(ctx, t, clientPool, CID) })
|
||||
t.Run("get zip "+version, func(t *testing.T) { getZip(ctx, t, clientPool, CID) })
|
||||
t.Run("simple put "+version, func(t *testing.T) { simplePut(ctx, t, clientPool, CID, version) })
|
||||
t.Run("simple get "+version, func(t *testing.T) { simpleGet(ctx, t, clientPool, CID, version) })
|
||||
t.Run("get by attribute "+version, func(t *testing.T) { getByAttr(ctx, t, clientPool, CID, version) })
|
||||
t.Run("get zip "+version, func(t *testing.T) { getZip(ctx, t, clientPool, CID, version) })
|
||||
|
||||
cancel()
|
||||
err = aioContainer.Terminate(ctx)
|
||||
|
@ -74,10 +85,19 @@ func runServer() context.CancelFunc {
|
|||
return cancel
|
||||
}
|
||||
|
||||
func simplePut(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *cid.ID) {
|
||||
func simplePut(ctx context.Context, t *testing.T, p *pool.Pool, CID *cid.ID, version string) {
|
||||
url := "http://localhost:8082/upload/" + CID.String()
|
||||
makePutRequestAndCheck(ctx, t, p, CID, url)
|
||||
|
||||
if version >= versionWithNativeNames {
|
||||
url = "http://localhost:8082/upload/" + testContainerName
|
||||
makePutRequestAndCheck(ctx, t, p, CID, url)
|
||||
}
|
||||
}
|
||||
|
||||
func makePutRequestAndCheck(ctx context.Context, t *testing.T, p *pool.Pool, cnrID *cid.ID, url string) {
|
||||
content := "content of file"
|
||||
keyAttr, valAttr := "User-Attribute", "user value"
|
||||
|
||||
attributes := map[string]string{
|
||||
object.AttributeFileName: "newFile.txt",
|
||||
keyAttr: valAttr,
|
||||
|
@ -92,23 +112,32 @@ func simplePut(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *ci
|
|||
err = w.Close()
|
||||
require.NoError(t, err)
|
||||
|
||||
request, err := http.NewRequest(http.MethodPost, "http://localhost:8082/upload/"+CID.String(), &buff)
|
||||
request, err := http.NewRequest(http.MethodPost, url, &buff)
|
||||
require.NoError(t, err)
|
||||
request.Header.Set("Content-Type", w.FormDataContentType())
|
||||
request.Header.Set("X-Attribute-"+keyAttr, valAttr)
|
||||
|
||||
resp, err := http.DefaultClient.Do(request)
|
||||
require.NoError(t, err)
|
||||
|
||||
defer func() {
|
||||
err = resp.Body.Close()
|
||||
err := resp.Body.Close()
|
||||
require.NoError(t, err)
|
||||
}()
|
||||
|
||||
addr := &putResponse{}
|
||||
err = json.NewDecoder(resp.Body).Decode(addr)
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = CID.DecodeString(addr.CID)
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
fmt.Println(string(body))
|
||||
}
|
||||
require.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
|
||||
addr := &putResponse{}
|
||||
err = json.Unmarshal(body, addr)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = cnrID.DecodeString(addr.CID)
|
||||
require.NoError(t, err)
|
||||
|
||||
id := new(oid.ID)
|
||||
|
@ -116,7 +145,7 @@ func simplePut(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *ci
|
|||
require.NoError(t, err)
|
||||
|
||||
objectAddress := address.NewAddress()
|
||||
objectAddress.SetContainerID(*CID)
|
||||
objectAddress.SetContainerID(*cnrID)
|
||||
objectAddress.SetObjectID(*id)
|
||||
|
||||
payload := bytes.NewBuffer(nil)
|
||||
|
@ -124,7 +153,7 @@ func simplePut(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *ci
|
|||
var prm pool.PrmObjectGet
|
||||
prm.SetAddress(*objectAddress)
|
||||
|
||||
res, err := clientPool.GetObject(ctx, prm)
|
||||
res, err := p.GetObject(ctx, prm)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = io.Copy(payload, res.Payload)
|
||||
|
@ -137,7 +166,7 @@ func simplePut(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *ci
|
|||
}
|
||||
}
|
||||
|
||||
func simpleGet(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *cid.ID) {
|
||||
func simpleGet(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *cid.ID, version string) {
|
||||
content := "content of file"
|
||||
attributes := map[string]string{
|
||||
"some-attr": "some-get-value",
|
||||
|
@ -147,8 +176,18 @@ func simpleGet(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *ci
|
|||
|
||||
resp, err := http.Get("http://localhost:8082/get/" + CID.String() + "/" + id.String())
|
||||
require.NoError(t, err)
|
||||
checkGetResponse(t, resp, content, attributes)
|
||||
|
||||
if version >= versionWithNativeNames {
|
||||
resp, err = http.Get("http://localhost:8082/get/" + testContainerName + "/" + id.String())
|
||||
require.NoError(t, err)
|
||||
checkGetResponse(t, resp, content, attributes)
|
||||
}
|
||||
}
|
||||
|
||||
func checkGetResponse(t *testing.T, resp *http.Response, content string, attributes map[string]string) {
|
||||
defer func() {
|
||||
err = resp.Body.Close()
|
||||
err := resp.Body.Close()
|
||||
require.NoError(t, err)
|
||||
}()
|
||||
|
||||
|
@ -161,7 +200,7 @@ func simpleGet(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *ci
|
|||
}
|
||||
}
|
||||
|
||||
func getByAttr(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *cid.ID) {
|
||||
func getByAttr(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *cid.ID, version string) {
|
||||
keyAttr, valAttr := "some-attr", "some-get-by-attr-value"
|
||||
content := "content of file"
|
||||
attributes := map[string]string{keyAttr: valAttr}
|
||||
|
@ -176,8 +215,18 @@ func getByAttr(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *ci
|
|||
|
||||
resp, err := http.Get("http://localhost:8082/get_by_attribute/" + CID.String() + "/" + keyAttr + "/" + valAttr)
|
||||
require.NoError(t, err)
|
||||
checkGetByAttrResponse(t, resp, content, expectedAttr)
|
||||
|
||||
if version >= versionWithNativeNames {
|
||||
resp, err = http.Get("http://localhost:8082/get_by_attribute/" + testContainerName + "/" + keyAttr + "/" + valAttr)
|
||||
require.NoError(t, err)
|
||||
checkGetByAttrResponse(t, resp, content, expectedAttr)
|
||||
}
|
||||
}
|
||||
|
||||
func checkGetByAttrResponse(t *testing.T, resp *http.Response, content string, attributes map[string]string) {
|
||||
defer func() {
|
||||
err = resp.Body.Close()
|
||||
err := resp.Body.Close()
|
||||
require.NoError(t, err)
|
||||
}()
|
||||
|
||||
|
@ -185,12 +234,12 @@ func getByAttr(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *ci
|
|||
require.NoError(t, err)
|
||||
require.Equal(t, content, string(data))
|
||||
|
||||
for k, v := range expectedAttr {
|
||||
for k, v := range attributes {
|
||||
require.Equal(t, v, resp.Header.Get(k))
|
||||
}
|
||||
}
|
||||
|
||||
func getZip(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *cid.ID) {
|
||||
func getZip(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *cid.ID, version string) {
|
||||
names := []string{"zipfolder/dir/name1.txt", "zipfolder/name2.txt"}
|
||||
contents := []string{"content of file1", "content of file2"}
|
||||
attributes1 := map[string]string{attributeFilePath: names[0]}
|
||||
|
@ -199,28 +248,35 @@ func getZip(ctx context.Context, t *testing.T, clientPool *pool.Pool, CID *cid.I
|
|||
putObject(ctx, t, clientPool, CID, contents[0], attributes1)
|
||||
putObject(ctx, t, clientPool, CID, contents[1], attributes2)
|
||||
|
||||
resp, err := http.Get("http://localhost:8082/zip/" + CID.String() + "/zipfolder")
|
||||
baseURL := "http://localhost:8082/zip/" + CID.String()
|
||||
makeZipTest(t, baseURL, names, contents)
|
||||
|
||||
if version >= versionWithNativeNames {
|
||||
baseURL = "http://localhost:8082/zip/" + testContainerName
|
||||
makeZipTest(t, baseURL, names, contents)
|
||||
}
|
||||
}
|
||||
|
||||
func makeZipTest(t *testing.T, baseURL string, names, contents []string) {
|
||||
url := baseURL + "/zipfolder"
|
||||
makeZipRequest(t, url, names, contents)
|
||||
|
||||
// check nested folder
|
||||
url = baseURL + "/zipfolder/dir"
|
||||
makeZipRequest(t, url, names[:1], contents[:1])
|
||||
}
|
||||
|
||||
func makeZipRequest(t *testing.T, url string, names, contents []string) {
|
||||
resp, err := http.Get(url)
|
||||
require.NoError(t, err)
|
||||
defer func() {
|
||||
err = resp.Body.Close()
|
||||
err := resp.Body.Close()
|
||||
require.NoError(t, err)
|
||||
}()
|
||||
|
||||
data, err := io.ReadAll(resp.Body)
|
||||
require.NoError(t, err)
|
||||
checkZip(t, data, resp.ContentLength, names, contents)
|
||||
|
||||
// check nested folder
|
||||
resp2, err := http.Get("http://localhost:8082/zip/" + CID.String() + "/zipfolder/dir")
|
||||
require.NoError(t, err)
|
||||
defer func() {
|
||||
err = resp2.Body.Close()
|
||||
require.NoError(t, err)
|
||||
}()
|
||||
|
||||
data2, err := io.ReadAll(resp2.Body)
|
||||
require.NoError(t, err)
|
||||
checkZip(t, data2, resp2.ContentLength, names[:1], contents[:1])
|
||||
}
|
||||
|
||||
func checkZip(t *testing.T, data []byte, length int64, names, contents []string) {
|
||||
|
@ -273,6 +329,8 @@ func getDefaultConfig() *viper.Viper {
|
|||
v.SetDefault(cfgPeers+".0.weight", 1)
|
||||
v.SetDefault(cfgPeers+".0.priority", 1)
|
||||
|
||||
v.SetDefault(cfgRPCEndpoint, "http://127.0.0.1:30333")
|
||||
|
||||
return v
|
||||
}
|
||||
|
||||
|
@ -290,17 +348,20 @@ func getPool(ctx context.Context, t *testing.T, key *keys.PrivateKey) *pool.Pool
|
|||
return clientPool
|
||||
}
|
||||
|
||||
func createContainer(ctx context.Context, t *testing.T, clientPool *pool.Pool) (*cid.ID, error) {
|
||||
func createContainer(ctx context.Context, t *testing.T, clientPool *pool.Pool, version string) (*cid.ID, error) {
|
||||
pp, err := policy.Parse("REP 1")
|
||||
require.NoError(t, err)
|
||||
|
||||
cnr := container.New(
|
||||
container.WithPolicy(pp),
|
||||
container.WithCustomBasicACL(0x0FFFFFFF),
|
||||
container.WithAttribute(container.AttributeName, "friendlyName"),
|
||||
container.WithAttribute(container.AttributeTimestamp, strconv.FormatInt(time.Now().Unix(), 10)))
|
||||
cnr.SetOwnerID(clientPool.OwnerID())
|
||||
|
||||
if version >= versionWithNativeNames {
|
||||
container.SetNativeName(cnr, testContainerName)
|
||||
}
|
||||
|
||||
var waitPrm pool.WaitParams
|
||||
waitPrm.SetTimeout(15 * time.Second)
|
||||
waitPrm.SetPollInterval(3 * time.Second)
|
||||
|
|
46
resolver/neofs.go
Normal file
46
resolver/neofs.go
Normal file
|
@ -0,0 +1,46 @@
|
|||
package resolver
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/nspcc-dev/neofs-sdk-go/netmap"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/pool"
|
||||
)
|
||||
|
||||
// NeoFSResolver represents virtual connection to the NeoFS network.
|
||||
// It implements resolver.NeoFS.
|
||||
type NeoFSResolver struct {
|
||||
pool *pool.Pool
|
||||
}
|
||||
|
||||
// NewNeoFSResolver creates new NeoFSResolver using provided pool.Pool.
|
||||
func NewNeoFSResolver(p *pool.Pool) *NeoFSResolver {
|
||||
return &NeoFSResolver{pool: p}
|
||||
}
|
||||
|
||||
// SystemDNS implements resolver.NeoFS interface method.
|
||||
func (x *NeoFSResolver) SystemDNS(ctx context.Context) (string, error) {
|
||||
networkInfo, err := x.pool.NetworkInfo(ctx)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read network info via client: %w", err)
|
||||
}
|
||||
|
||||
var domain string
|
||||
|
||||
networkInfo.NetworkConfig().IterateParameters(func(parameter *netmap.NetworkParameter) bool {
|
||||
if string(parameter.Key()) == "SystemDNS" {
|
||||
domain = string(parameter.Value())
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
})
|
||||
|
||||
if domain == "" {
|
||||
return "", errors.New("system DNS parameter not found or empty")
|
||||
}
|
||||
|
||||
return domain, nil
|
||||
}
|
140
resolver/resolver.go
Normal file
140
resolver/resolver.go
Normal file
|
@ -0,0 +1,140 @@
|
|||
package resolver
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/ns"
|
||||
)
|
||||
|
||||
const (
|
||||
NNSResolver = "nns"
|
||||
DNSResolver = "dns"
|
||||
)
|
||||
|
||||
// NeoFS represents virtual connection to the NeoFS network.
|
||||
type NeoFS interface {
|
||||
// SystemDNS reads system DNS network parameters of the NeoFS.
|
||||
//
|
||||
// Returns exactly on non-zero value. Returns any error encountered
|
||||
// which prevented the parameter to be read.
|
||||
SystemDNS(context.Context) (string, error)
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
NeoFS NeoFS
|
||||
RPCAddress string
|
||||
}
|
||||
|
||||
type ContainerResolver struct {
|
||||
Name string
|
||||
resolve func(context.Context, string) (*cid.ID, error)
|
||||
|
||||
next *ContainerResolver
|
||||
}
|
||||
|
||||
func (r *ContainerResolver) SetResolveFunc(fn func(context.Context, string) (*cid.ID, error)) {
|
||||
r.resolve = fn
|
||||
}
|
||||
|
||||
func (r *ContainerResolver) Resolve(ctx context.Context, name string) (*cid.ID, error) {
|
||||
cnrID, err := r.resolve(ctx, name)
|
||||
if err != nil {
|
||||
if r.next != nil {
|
||||
cnrID, inErr := r.next.Resolve(ctx, name)
|
||||
if inErr != nil {
|
||||
return nil, fmt.Errorf("%s; %w", err.Error(), inErr)
|
||||
}
|
||||
return cnrID, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return cnrID, nil
|
||||
}
|
||||
|
||||
func NewResolver(order []string, cfg *Config) (*ContainerResolver, error) {
|
||||
if len(order) == 0 {
|
||||
return nil, fmt.Errorf("resolving order must not be empty")
|
||||
}
|
||||
|
||||
bucketResolver, err := newResolver(order[len(order)-1], cfg, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for i := len(order) - 2; i >= 0; i-- {
|
||||
resolverName := order[i]
|
||||
next := bucketResolver
|
||||
|
||||
bucketResolver, err = newResolver(resolverName, cfg, next)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return bucketResolver, nil
|
||||
}
|
||||
|
||||
func newResolver(name string, cfg *Config, next *ContainerResolver) (*ContainerResolver, error) {
|
||||
switch name {
|
||||
case DNSResolver:
|
||||
return NewDNSResolver(cfg.NeoFS, next)
|
||||
case NNSResolver:
|
||||
return NewNNSResolver(cfg.RPCAddress, next)
|
||||
default:
|
||||
return nil, fmt.Errorf("unknown resolver: %s", name)
|
||||
}
|
||||
}
|
||||
|
||||
func NewDNSResolver(neoFS NeoFS, next *ContainerResolver) (*ContainerResolver, error) {
|
||||
if neoFS == nil {
|
||||
return nil, fmt.Errorf("pool must not be nil for DNS resolver")
|
||||
}
|
||||
|
||||
var dns ns.DNS
|
||||
|
||||
resolveFunc := func(ctx context.Context, name string) (*cid.ID, error) {
|
||||
domain, err := neoFS.SystemDNS(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read system DNS parameter of the NeoFS: %w", err)
|
||||
}
|
||||
|
||||
domain = name + "." + domain
|
||||
cnrID, err := dns.ResolveContainerName(domain)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't resolve container '%s' as '%s': %w", name, domain, err)
|
||||
}
|
||||
return &cnrID, nil
|
||||
}
|
||||
|
||||
return &ContainerResolver{
|
||||
Name: DNSResolver,
|
||||
|
||||
resolve: resolveFunc,
|
||||
next: next,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func NewNNSResolver(rpcAddress string, next *ContainerResolver) (*ContainerResolver, error) {
|
||||
var nns ns.NNS
|
||||
|
||||
if err := nns.Dial(rpcAddress); err != nil {
|
||||
return nil, fmt.Errorf("could not dial nns: %w", err)
|
||||
}
|
||||
|
||||
resolveFunc := func(_ context.Context, name string) (*cid.ID, error) {
|
||||
cnrID, err := nns.ResolveContainerName(name)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't resolve container '%s': %w", name, err)
|
||||
}
|
||||
return &cnrID, nil
|
||||
}
|
||||
|
||||
return &ContainerResolver{
|
||||
Name: NNSResolver,
|
||||
|
||||
resolve: resolveFunc,
|
||||
next: next,
|
||||
}, nil
|
||||
}
|
13
settings.go
13
settings.go
|
@ -8,6 +8,7 @@ import (
|
|||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nspcc-dev/neofs-http-gw/resolver"
|
||||
"github.com/spf13/pflag"
|
||||
"github.com/spf13/viper"
|
||||
"github.com/valyala/fasthttp"
|
||||
|
@ -49,6 +50,12 @@ const (
|
|||
// Peers.
|
||||
cfgPeers = "peers"
|
||||
|
||||
// NeoGo.
|
||||
cfgRPCEndpoint = "rpc_endpoint"
|
||||
|
||||
// Resolving.
|
||||
cfgResolveOrder = "resolve_order"
|
||||
|
||||
// Zip compression.
|
||||
cfgZipCompression = "zip.compression"
|
||||
|
||||
|
@ -99,6 +106,8 @@ func settings() *viper.Viper {
|
|||
flags.String(cfgTLSKey, "", "TLS key path")
|
||||
peers := flags.StringArrayP(cfgPeers, "p", nil, "NeoFS nodes")
|
||||
|
||||
resolveMethods := flags.StringSlice(cfgResolveOrder, []string{resolver.NNSResolver, resolver.DNSResolver}, "set container name resolve order")
|
||||
|
||||
// set defaults:
|
||||
|
||||
// logger:
|
||||
|
@ -126,6 +135,10 @@ func settings() *viper.Viper {
|
|||
panic(err)
|
||||
}
|
||||
|
||||
if resolveMethods != nil {
|
||||
v.SetDefault(cfgResolveOrder, *resolveMethods)
|
||||
}
|
||||
|
||||
switch {
|
||||
case help != nil && *help:
|
||||
fmt.Printf("NeoFS HTTP Gateway %s\n", Version)
|
||||
|
|
|
@ -9,11 +9,11 @@ import (
|
|||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/nspcc-dev/neofs-http-gw/resolver"
|
||||
"github.com/nspcc-dev/neofs-http-gw/response"
|
||||
"github.com/nspcc-dev/neofs-http-gw/tokens"
|
||||
"github.com/nspcc-dev/neofs-http-gw/utils"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/bearer"
|
||||
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/netmap"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/object"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/object/address"
|
||||
|
@ -35,6 +35,7 @@ type Uploader struct {
|
|||
log *zap.Logger
|
||||
pool *pool.Pool
|
||||
enableDefaultTimestamp bool
|
||||
containerResolver *resolver.ContainerResolver
|
||||
}
|
||||
|
||||
type epochDurations struct {
|
||||
|
@ -45,33 +46,41 @@ type epochDurations struct {
|
|||
|
||||
// New creates a new Uploader using specified logger, connection pool and
|
||||
// other options.
|
||||
func New(ctx context.Context, log *zap.Logger, conns *pool.Pool, enableDefaultTimestamp bool) *Uploader {
|
||||
return &Uploader{ctx, log, conns, enableDefaultTimestamp}
|
||||
func New(ctx context.Context, params *utils.AppParams, enableDefaultTimestamp bool) *Uploader {
|
||||
return &Uploader{
|
||||
appCtx: ctx,
|
||||
log: params.Logger,
|
||||
pool: params.Pool,
|
||||
enableDefaultTimestamp: enableDefaultTimestamp,
|
||||
containerResolver: params.Resolver,
|
||||
}
|
||||
}
|
||||
|
||||
// Upload handles multipart upload request.
|
||||
func (u *Uploader) Upload(c *fasthttp.RequestCtx) {
|
||||
var (
|
||||
err error
|
||||
file MultipartFile
|
||||
idObj *oid.ID
|
||||
addr = address.NewAddress()
|
||||
idCnr = new(cid.ID)
|
||||
scid, _ = c.UserValue("cid").(string)
|
||||
log = u.log.With(zap.String("cid", scid))
|
||||
bodyStream = c.RequestBodyStream()
|
||||
drainBuf = make([]byte, drainBufSize)
|
||||
)
|
||||
if err = tokens.StoreBearerToken(c); err != nil {
|
||||
|
||||
if err := tokens.StoreBearerToken(c); err != nil {
|
||||
log.Error("could not fetch bearer token", zap.Error(err))
|
||||
response.Error(c, "could not fetch bearer token", fasthttp.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err = idCnr.DecodeString(scid); err != nil {
|
||||
|
||||
idCnr, err := utils.GetContainerID(u.appCtx, scid, u.containerResolver)
|
||||
if err != nil {
|
||||
log.Error("wrong container id", zap.Error(err))
|
||||
response.Error(c, "wrong container id", fasthttp.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
defer func() {
|
||||
// If the temporary reader can be closed - let's close it.
|
||||
if file == nil {
|
||||
|
|
13
utils/params.go
Normal file
13
utils/params.go
Normal file
|
@ -0,0 +1,13 @@
|
|||
package utils
|
||||
|
||||
import (
|
||||
"github.com/nspcc-dev/neofs-http-gw/resolver"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/pool"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
type AppParams struct {
|
||||
Logger *zap.Logger
|
||||
Pool *pool.Pool
|
||||
Resolver *resolver.ContainerResolver
|
||||
}
|
21
utils/util.go
Normal file
21
utils/util.go
Normal file
|
@ -0,0 +1,21 @@
|
|||
package utils
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/nspcc-dev/neofs-http-gw/resolver"
|
||||
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
|
||||
)
|
||||
|
||||
// GetContainerID decode container id, if it's not a valid container id
|
||||
// then trey to resolve name using provided resolver.
|
||||
func GetContainerID(ctx context.Context, containerID string, resolver *resolver.ContainerResolver) (*cid.ID, error) {
|
||||
cnrID := new(cid.ID)
|
||||
err := cnrID.DecodeString(containerID)
|
||||
if err != nil {
|
||||
if resolver != nil {
|
||||
cnrID, err = resolver.Resolve(ctx, containerID)
|
||||
}
|
||||
}
|
||||
return cnrID, err
|
||||
}
|
Loading…
Reference in a new issue