Roman Loginov
ef2b75597c
Fallback path to search is needed because some software may keep FileName attribute and ignore FilePath attribute during file upload. Therefore, if this feature is enabled under certain conditions (for more information, see gate-configuration.md) a search will be performed for the FileName attribute. Signed-off-by: Roman Loginov <r.loginov@yadro.com>
481 lines
13 KiB
Go
481 lines
13 KiB
Go
package handler
|
|
|
|
import (
|
|
"archive/zip"
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"io"
|
|
"mime/multipart"
|
|
"net/http"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl"
|
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
|
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
|
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
|
"github.com/panjf2000/ants/v2"
|
|
"github.com/stretchr/testify/require"
|
|
"github.com/valyala/fasthttp"
|
|
"go.uber.org/zap"
|
|
)
|
|
|
|
type treeClientMock struct {
|
|
}
|
|
|
|
func (t *treeClientMock) GetNodes(context.Context, *tree.GetNodesParams) ([]tree.NodeResponse, error) {
|
|
return nil, nil
|
|
}
|
|
|
|
func (t *treeClientMock) GetSubTree(context.Context, *data.BucketInfo, string, []uint64, uint32, bool) ([]tree.NodeResponse, error) {
|
|
return nil, nil
|
|
}
|
|
|
|
type configMock struct {
|
|
additionalSearch bool
|
|
}
|
|
|
|
func (c *configMock) DefaultTimestamp() bool {
|
|
return false
|
|
}
|
|
|
|
func (c *configMock) ZipCompression() bool {
|
|
return false
|
|
}
|
|
|
|
func (c *configMock) IndexPageEnabled() bool {
|
|
return false
|
|
}
|
|
|
|
func (c *configMock) IndexPageTemplate() string {
|
|
return ""
|
|
}
|
|
|
|
func (c *configMock) IndexPageNativeTemplate() string {
|
|
return ""
|
|
}
|
|
|
|
func (c *configMock) ClientCut() bool {
|
|
return false
|
|
}
|
|
|
|
func (c *configMock) BufferMaxSizeForPut() uint64 {
|
|
return 0
|
|
}
|
|
|
|
func (c *configMock) NamespaceHeader() string {
|
|
return ""
|
|
}
|
|
|
|
func (c *configMock) EnableFilepathFallback() bool {
|
|
return c.additionalSearch
|
|
}
|
|
|
|
type handlerContext struct {
|
|
key *keys.PrivateKey
|
|
owner user.ID
|
|
|
|
h *Handler
|
|
frostfs *TestFrostFS
|
|
tree *treeClientMock
|
|
cfg *configMock
|
|
}
|
|
|
|
func (hc *handlerContext) Handler() *Handler {
|
|
return hc.h
|
|
}
|
|
|
|
func prepareHandlerContext() (*handlerContext, error) {
|
|
logger, err := zap.NewDevelopment()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
key, err := keys.NewPrivateKey()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
var owner user.ID
|
|
user.IDFromKey(&owner, key.PrivateKey.PublicKey)
|
|
|
|
testFrostFS := NewTestFrostFS(key)
|
|
|
|
testResolver := &resolver.Resolver{Name: "test_resolver"}
|
|
testResolver.SetResolveFunc(func(_ context.Context, name string) (*cid.ID, error) {
|
|
return testFrostFS.ContainerID(name)
|
|
})
|
|
|
|
params := &AppParams{
|
|
Logger: logger,
|
|
FrostFS: testFrostFS,
|
|
Owner: &owner,
|
|
Resolver: testResolver,
|
|
Cache: cache.NewBucketCache(&cache.Config{
|
|
Size: 1,
|
|
Lifetime: 1,
|
|
Logger: logger,
|
|
}),
|
|
}
|
|
|
|
treeMock := &treeClientMock{}
|
|
cfgMock := &configMock{}
|
|
|
|
workerPool, err := ants.NewPool(1000)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
handler := New(params, cfgMock, tree.NewTree(treeMock), workerPool)
|
|
|
|
return &handlerContext{
|
|
key: key,
|
|
owner: owner,
|
|
h: handler,
|
|
frostfs: testFrostFS,
|
|
tree: treeMock,
|
|
cfg: cfgMock,
|
|
}, nil
|
|
}
|
|
|
|
func (hc *handlerContext) prepareContainer(name string, basicACL acl.Basic) (cid.ID, *container.Container, error) {
|
|
var pp netmap.PlacementPolicy
|
|
err := pp.DecodeString("REP 1")
|
|
if err != nil {
|
|
return cid.ID{}, nil, err
|
|
}
|
|
|
|
var cnr container.Container
|
|
cnr.Init()
|
|
cnr.SetOwner(hc.owner)
|
|
cnr.SetPlacementPolicy(pp)
|
|
cnr.SetBasicACL(basicACL)
|
|
|
|
var domain container.Domain
|
|
domain.SetName(name)
|
|
container.WriteDomain(&cnr, domain)
|
|
container.SetName(&cnr, name)
|
|
container.SetCreationTime(&cnr, time.Now())
|
|
|
|
cnrID := cidtest.ID()
|
|
|
|
for op := acl.OpObjectGet; op < acl.OpObjectHash; op++ {
|
|
hc.frostfs.AllowUserOperation(cnrID, hc.owner, op, oid.ID{})
|
|
if basicACL.IsOpAllowed(op, acl.RoleOthers) {
|
|
hc.frostfs.AllowUserOperation(cnrID, user.ID{}, op, oid.ID{})
|
|
}
|
|
}
|
|
|
|
return cnrID, &cnr, nil
|
|
}
|
|
|
|
func TestBasic(t *testing.T) {
|
|
hc, err := prepareHandlerContext()
|
|
require.NoError(t, err)
|
|
|
|
bktName := "bucket"
|
|
cnrID, cnr, err := hc.prepareContainer(bktName, acl.PublicRWExtended)
|
|
require.NoError(t, err)
|
|
hc.frostfs.SetContainer(cnrID, cnr)
|
|
|
|
ctx := context.Background()
|
|
ctx = middleware.SetNamespace(ctx, "")
|
|
|
|
content := "hello"
|
|
r, err := prepareUploadRequest(ctx, cnrID.EncodeToString(), content)
|
|
require.NoError(t, err)
|
|
|
|
hc.Handler().Upload(r)
|
|
require.Equal(t, r.Response.StatusCode(), http.StatusOK)
|
|
|
|
var putRes putResponse
|
|
err = json.Unmarshal(r.Response.Body(), &putRes)
|
|
require.NoError(t, err)
|
|
|
|
obj := hc.frostfs.objects[putRes.ContainerID+"/"+putRes.ObjectID]
|
|
attr := prepareObjectAttributes(object.AttributeFilePath, objFileName)
|
|
obj.SetAttributes(append(obj.Attributes(), attr)...)
|
|
|
|
t.Run("get", func(t *testing.T) {
|
|
r = prepareGetRequest(ctx, cnrID.EncodeToString(), putRes.ObjectID)
|
|
hc.Handler().DownloadByAddressOrBucketName(r)
|
|
require.Equal(t, content, string(r.Response.Body()))
|
|
})
|
|
|
|
t.Run("head", func(t *testing.T) {
|
|
r = prepareGetRequest(ctx, cnrID.EncodeToString(), putRes.ObjectID)
|
|
hc.Handler().HeadByAddressOrBucketName(r)
|
|
require.Equal(t, putRes.ObjectID, string(r.Response.Header.Peek(hdrObjectID)))
|
|
require.Equal(t, putRes.ContainerID, string(r.Response.Header.Peek(hdrContainerID)))
|
|
})
|
|
|
|
t.Run("get by attribute", func(t *testing.T) {
|
|
r = prepareGetByAttributeRequest(ctx, bktName, keyAttr, valAttr)
|
|
hc.Handler().DownloadByAttribute(r)
|
|
require.Equal(t, content, string(r.Response.Body()))
|
|
})
|
|
|
|
t.Run("head by attribute", func(t *testing.T) {
|
|
r = prepareGetByAttributeRequest(ctx, bktName, keyAttr, valAttr)
|
|
hc.Handler().HeadByAttribute(r)
|
|
require.Equal(t, putRes.ObjectID, string(r.Response.Header.Peek(hdrObjectID)))
|
|
require.Equal(t, putRes.ContainerID, string(r.Response.Header.Peek(hdrContainerID)))
|
|
})
|
|
|
|
t.Run("zip", func(t *testing.T) {
|
|
r = prepareGetZipped(ctx, bktName, "")
|
|
hc.Handler().DownloadZipped(r)
|
|
|
|
readerAt := bytes.NewReader(r.Response.Body())
|
|
zipReader, err := zip.NewReader(readerAt, int64(len(r.Response.Body())))
|
|
require.NoError(t, err)
|
|
require.Len(t, zipReader.File, 1)
|
|
require.Equal(t, objFileName, zipReader.File[0].Name)
|
|
f, err := zipReader.File[0].Open()
|
|
require.NoError(t, err)
|
|
defer func() {
|
|
inErr := f.Close()
|
|
require.NoError(t, inErr)
|
|
}()
|
|
data, err := io.ReadAll(f)
|
|
require.NoError(t, err)
|
|
require.Equal(t, content, string(data))
|
|
})
|
|
}
|
|
|
|
func TestFindObjectByAttribute(t *testing.T) {
|
|
hc, err := prepareHandlerContext()
|
|
require.NoError(t, err)
|
|
hc.cfg.additionalSearch = true
|
|
|
|
bktName := "bucket"
|
|
cnrID, cnr, err := hc.prepareContainer(bktName, acl.PublicRWExtended)
|
|
require.NoError(t, err)
|
|
hc.frostfs.SetContainer(cnrID, cnr)
|
|
|
|
ctx := context.Background()
|
|
ctx = middleware.SetNamespace(ctx, "")
|
|
|
|
content := "hello"
|
|
r, err := prepareUploadRequest(ctx, cnrID.EncodeToString(), content)
|
|
require.NoError(t, err)
|
|
|
|
hc.Handler().Upload(r)
|
|
require.Equal(t, r.Response.StatusCode(), http.StatusOK)
|
|
|
|
var putRes putResponse
|
|
err = json.Unmarshal(r.Response.Body(), &putRes)
|
|
require.NoError(t, err)
|
|
|
|
testAttrVal1 := "test-attr-val1"
|
|
testAttrVal2 := "test-attr-val2"
|
|
testAttrVal3 := "test-attr-val3"
|
|
|
|
for _, tc := range []struct {
|
|
name string
|
|
firstAttr object.Attribute
|
|
secondAttr object.Attribute
|
|
reqAttrKey string
|
|
reqAttrValue string
|
|
err string
|
|
additionalSearch bool
|
|
}{
|
|
{
|
|
name: "success search by FileName",
|
|
firstAttr: prepareObjectAttributes(attrFilePath, testAttrVal1),
|
|
secondAttr: prepareObjectAttributes(attrFileName, testAttrVal2),
|
|
reqAttrKey: attrFileName,
|
|
reqAttrValue: testAttrVal2,
|
|
additionalSearch: false,
|
|
},
|
|
{
|
|
name: "failed search by FileName",
|
|
firstAttr: prepareObjectAttributes(attrFilePath, testAttrVal1),
|
|
secondAttr: prepareObjectAttributes(attrFileName, testAttrVal2),
|
|
reqAttrKey: attrFileName,
|
|
reqAttrValue: testAttrVal3,
|
|
err: "not found",
|
|
additionalSearch: false,
|
|
},
|
|
{
|
|
name: "success search by FilePath (with additional search)",
|
|
firstAttr: prepareObjectAttributes(attrFilePath, testAttrVal1),
|
|
secondAttr: prepareObjectAttributes(attrFileName, testAttrVal2),
|
|
reqAttrKey: attrFilePath,
|
|
reqAttrValue: testAttrVal2,
|
|
additionalSearch: true,
|
|
},
|
|
{
|
|
name: "failed by FilePath (with additional search)",
|
|
firstAttr: prepareObjectAttributes(attrFilePath, testAttrVal1),
|
|
secondAttr: prepareObjectAttributes(attrFileName, testAttrVal2),
|
|
reqAttrKey: attrFilePath,
|
|
reqAttrValue: testAttrVal3,
|
|
err: "not found",
|
|
additionalSearch: true,
|
|
},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
obj := hc.frostfs.objects[putRes.ContainerID+"/"+putRes.ObjectID]
|
|
obj.SetAttributes(tc.firstAttr, tc.secondAttr)
|
|
hc.cfg.additionalSearch = tc.additionalSearch
|
|
|
|
objID, err := hc.Handler().findObjectByAttribute(ctx, hc.Handler().log, cnrID, tc.reqAttrKey, tc.reqAttrValue)
|
|
if tc.err != "" {
|
|
require.Error(t, err)
|
|
require.Contains(t, err.Error(), tc.err)
|
|
return
|
|
}
|
|
|
|
require.NoError(t, err)
|
|
require.Equal(t, putRes.ObjectID, objID.EncodeToString())
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestNeedSearchByFileName(t *testing.T) {
|
|
hc, err := prepareHandlerContext()
|
|
require.NoError(t, err)
|
|
|
|
for _, tc := range []struct {
|
|
name string
|
|
attrKey string
|
|
attrVal string
|
|
additionalSearch bool
|
|
expected bool
|
|
}{
|
|
{
|
|
name: "need search - not contains slash",
|
|
attrKey: attrFilePath,
|
|
attrVal: "cat.png",
|
|
additionalSearch: true,
|
|
expected: true,
|
|
},
|
|
{
|
|
name: "need search - single lead slash",
|
|
attrKey: attrFilePath,
|
|
attrVal: "/cat.png",
|
|
additionalSearch: true,
|
|
expected: true,
|
|
},
|
|
{
|
|
name: "don't need search - single slash but not lead",
|
|
attrKey: attrFilePath,
|
|
attrVal: "cats/cat.png",
|
|
additionalSearch: true,
|
|
expected: false,
|
|
},
|
|
{
|
|
name: "don't need search - more one slash",
|
|
attrKey: attrFilePath,
|
|
attrVal: "/cats/cat.png",
|
|
additionalSearch: true,
|
|
expected: false,
|
|
},
|
|
{
|
|
name: "don't need search - incorrect attribute key",
|
|
attrKey: attrFileName,
|
|
attrVal: "cat.png",
|
|
additionalSearch: true,
|
|
expected: false,
|
|
},
|
|
{
|
|
name: "don't need search - additional search disabled",
|
|
attrKey: attrFilePath,
|
|
attrVal: "cat.png",
|
|
additionalSearch: false,
|
|
expected: false,
|
|
},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
hc.cfg.additionalSearch = tc.additionalSearch
|
|
|
|
res := hc.h.needSearchByFileName(tc.attrKey, tc.attrVal)
|
|
require.Equal(t, tc.expected, res)
|
|
})
|
|
}
|
|
}
|
|
|
|
func prepareUploadRequest(ctx context.Context, bucket, content string) (*fasthttp.RequestCtx, error) {
|
|
r := new(fasthttp.RequestCtx)
|
|
utils.SetContextToRequest(ctx, r)
|
|
r.SetUserValue("cid", bucket)
|
|
return r, fillMultipartBody(r, content)
|
|
}
|
|
|
|
func prepareGetRequest(ctx context.Context, bucket, objID string) *fasthttp.RequestCtx {
|
|
r := new(fasthttp.RequestCtx)
|
|
utils.SetContextToRequest(ctx, r)
|
|
r.SetUserValue("cid", bucket)
|
|
r.SetUserValue("oid", objID)
|
|
return r
|
|
}
|
|
|
|
func prepareGetByAttributeRequest(ctx context.Context, bucket, attrKey, attrVal string) *fasthttp.RequestCtx {
|
|
r := new(fasthttp.RequestCtx)
|
|
utils.SetContextToRequest(ctx, r)
|
|
r.SetUserValue("cid", bucket)
|
|
r.SetUserValue("attr_key", attrKey)
|
|
r.SetUserValue("attr_val", attrVal)
|
|
return r
|
|
}
|
|
|
|
func prepareGetZipped(ctx context.Context, bucket, prefix string) *fasthttp.RequestCtx {
|
|
r := new(fasthttp.RequestCtx)
|
|
utils.SetContextToRequest(ctx, r)
|
|
r.SetUserValue("cid", bucket)
|
|
r.SetUserValue("prefix", prefix)
|
|
return r
|
|
}
|
|
|
|
func prepareObjectAttributes(attrKey, attrValue string) object.Attribute {
|
|
attr := object.NewAttribute()
|
|
attr.SetKey(attrKey)
|
|
attr.SetValue(attrValue)
|
|
return *attr
|
|
}
|
|
|
|
const (
|
|
keyAttr = "User-Attribute"
|
|
valAttr = "user value"
|
|
objFileName = "newFile.txt"
|
|
)
|
|
|
|
func fillMultipartBody(r *fasthttp.RequestCtx, content string) error {
|
|
attributes := map[string]string{
|
|
object.AttributeFileName: objFileName,
|
|
keyAttr: valAttr,
|
|
}
|
|
|
|
var buff bytes.Buffer
|
|
w := multipart.NewWriter(&buff)
|
|
fw, err := w.CreateFormFile("file", attributes[object.AttributeFileName])
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if _, err = io.Copy(fw, bytes.NewBufferString(content)); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err = w.Close(); err != nil {
|
|
return err
|
|
}
|
|
|
|
r.Request.SetBodyStream(&buff, buff.Len())
|
|
r.Request.Header.Set("Content-Type", w.FormDataContentType())
|
|
r.Request.Header.Set("X-Attribute-"+keyAttr, valAttr)
|
|
|
|
return nil
|
|
}
|