Nikita Zinkevich
1be92fa4be
Some checks failed
/ DCO (pull_request) Successful in 4m21s
/ Vulncheck (pull_request) Successful in 4m31s
/ Builds (pull_request) Successful in 1m35s
/ Lint (pull_request) Successful in 2m56s
/ Tests (pull_request) Successful in 1m57s
/ Builds (push) Has been cancelled
/ Lint (push) Has been cancelled
/ Tests (push) Has been cancelled
/ Vulncheck (push) Has been cancelled
Prioritize getting s3 object with the key, which equals to valid FrostFS OID, rather than getting non-existent object with OID via native protocol for GET and HEAD requests Signed-off-by: Nikita Zinkevich <n.zinkevich@yadro.com>
496 lines
13 KiB
Go
496 lines
13 KiB
Go
package handler
|
|
|
|
import (
|
|
"archive/zip"
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"io"
|
|
"mime/multipart"
|
|
"net/http"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl"
|
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
|
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
|
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
|
"github.com/panjf2000/ants/v2"
|
|
"github.com/stretchr/testify/require"
|
|
"github.com/valyala/fasthttp"
|
|
"go.uber.org/zap"
|
|
)
|
|
|
|
type treeServiceMock struct {
|
|
system map[string]map[string]*data.BaseNodeVersion
|
|
}
|
|
|
|
func newTreeService() *treeServiceMock {
|
|
return &treeServiceMock{
|
|
system: make(map[string]map[string]*data.BaseNodeVersion),
|
|
}
|
|
}
|
|
|
|
func (t *treeServiceMock) CheckSettingsNodeExists(context.Context, *data.BucketInfo) error {
|
|
_, ok := t.system["bucket-settings"]
|
|
if !ok {
|
|
return layer.ErrNodeNotFound
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (t *treeServiceMock) GetSubTreeByPrefix(context.Context, *data.BucketInfo, string, bool) ([]data.NodeInfo, string, error) {
|
|
return nil, "", nil
|
|
}
|
|
|
|
func (t *treeServiceMock) GetLatestVersion(context.Context, *cid.ID, string) (*data.NodeVersion, error) {
|
|
return nil, nil
|
|
}
|
|
|
|
type configMock struct {
|
|
additionalSearch bool
|
|
}
|
|
|
|
func (c *configMock) DefaultTimestamp() bool {
|
|
return false
|
|
}
|
|
|
|
func (c *configMock) ZipCompression() bool {
|
|
return false
|
|
}
|
|
|
|
func (c *configMock) IndexPageEnabled() bool {
|
|
return false
|
|
}
|
|
|
|
func (c *configMock) IndexPageTemplate() string {
|
|
return ""
|
|
}
|
|
|
|
func (c *configMock) IndexPageNativeTemplate() string {
|
|
return ""
|
|
}
|
|
|
|
func (c *configMock) ClientCut() bool {
|
|
return false
|
|
}
|
|
|
|
func (c *configMock) BufferMaxSizeForPut() uint64 {
|
|
return 0
|
|
}
|
|
|
|
func (c *configMock) NamespaceHeader() string {
|
|
return ""
|
|
}
|
|
|
|
func (c *configMock) EnableFilepathFallback() bool {
|
|
return c.additionalSearch
|
|
}
|
|
|
|
type handlerContext struct {
|
|
key *keys.PrivateKey
|
|
owner user.ID
|
|
|
|
h *Handler
|
|
frostfs *TestFrostFS
|
|
tree *treeServiceMock
|
|
cfg *configMock
|
|
}
|
|
|
|
func (hc *handlerContext) Handler() *Handler {
|
|
return hc.h
|
|
}
|
|
|
|
func prepareHandlerContext() (*handlerContext, error) {
|
|
logger, err := zap.NewDevelopment()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
key, err := keys.NewPrivateKey()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
var owner user.ID
|
|
user.IDFromKey(&owner, key.PrivateKey.PublicKey)
|
|
|
|
testFrostFS := NewTestFrostFS(key)
|
|
|
|
testResolver := &resolver.Resolver{Name: "test_resolver"}
|
|
testResolver.SetResolveFunc(func(_ context.Context, name string) (*cid.ID, error) {
|
|
return testFrostFS.ContainerID(name)
|
|
})
|
|
|
|
params := &AppParams{
|
|
Logger: logger,
|
|
FrostFS: testFrostFS,
|
|
Owner: &owner,
|
|
Resolver: testResolver,
|
|
Cache: cache.NewBucketCache(&cache.Config{
|
|
Size: 1,
|
|
Lifetime: 1,
|
|
Logger: logger,
|
|
}),
|
|
}
|
|
|
|
treeMock := newTreeService()
|
|
cfgMock := &configMock{}
|
|
|
|
workerPool, err := ants.NewPool(1)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
handler := New(params, cfgMock, treeMock, workerPool)
|
|
|
|
return &handlerContext{
|
|
key: key,
|
|
owner: owner,
|
|
h: handler,
|
|
frostfs: testFrostFS,
|
|
tree: treeMock,
|
|
cfg: cfgMock,
|
|
}, nil
|
|
}
|
|
|
|
func (hc *handlerContext) prepareContainer(name string, basicACL acl.Basic) (cid.ID, *container.Container, error) {
|
|
var pp netmap.PlacementPolicy
|
|
err := pp.DecodeString("REP 1")
|
|
if err != nil {
|
|
return cid.ID{}, nil, err
|
|
}
|
|
|
|
var cnr container.Container
|
|
cnr.Init()
|
|
cnr.SetOwner(hc.owner)
|
|
cnr.SetPlacementPolicy(pp)
|
|
cnr.SetBasicACL(basicACL)
|
|
|
|
var domain container.Domain
|
|
domain.SetName(name)
|
|
container.WriteDomain(&cnr, domain)
|
|
container.SetName(&cnr, name)
|
|
container.SetCreationTime(&cnr, time.Now())
|
|
|
|
cnrID := cidtest.ID()
|
|
|
|
for op := acl.OpObjectGet; op < acl.OpObjectHash; op++ {
|
|
hc.frostfs.AllowUserOperation(cnrID, hc.owner, op, oid.ID{})
|
|
if basicACL.IsOpAllowed(op, acl.RoleOthers) {
|
|
hc.frostfs.AllowUserOperation(cnrID, user.ID{}, op, oid.ID{})
|
|
}
|
|
}
|
|
|
|
return cnrID, &cnr, nil
|
|
}
|
|
|
|
func TestBasic(t *testing.T) {
|
|
hc, err := prepareHandlerContext()
|
|
require.NoError(t, err)
|
|
|
|
bktName := "bucket"
|
|
cnrID, cnr, err := hc.prepareContainer(bktName, acl.PublicRWExtended)
|
|
require.NoError(t, err)
|
|
hc.frostfs.SetContainer(cnrID, cnr)
|
|
|
|
ctx := context.Background()
|
|
ctx = middleware.SetNamespace(ctx, "")
|
|
|
|
content := "hello"
|
|
r, err := prepareUploadRequest(ctx, cnrID.EncodeToString(), content)
|
|
require.NoError(t, err)
|
|
|
|
hc.Handler().Upload(r)
|
|
require.Equal(t, r.Response.StatusCode(), http.StatusOK)
|
|
|
|
var putRes putResponse
|
|
err = json.Unmarshal(r.Response.Body(), &putRes)
|
|
require.NoError(t, err)
|
|
|
|
obj := hc.frostfs.objects[putRes.ContainerID+"/"+putRes.ObjectID]
|
|
attr := prepareObjectAttributes(object.AttributeFilePath, objFileName)
|
|
obj.SetAttributes(append(obj.Attributes(), attr)...)
|
|
|
|
t.Run("get", func(t *testing.T) {
|
|
r = prepareGetRequest(ctx, cnrID.EncodeToString(), putRes.ObjectID)
|
|
hc.Handler().DownloadByAddressOrBucketName(r)
|
|
require.Equal(t, content, string(r.Response.Body()))
|
|
})
|
|
|
|
t.Run("head", func(t *testing.T) {
|
|
r = prepareGetRequest(ctx, cnrID.EncodeToString(), putRes.ObjectID)
|
|
hc.Handler().HeadByAddressOrBucketName(r)
|
|
require.Equal(t, putRes.ObjectID, string(r.Response.Header.Peek(hdrObjectID)))
|
|
require.Equal(t, putRes.ContainerID, string(r.Response.Header.Peek(hdrContainerID)))
|
|
})
|
|
|
|
t.Run("get by attribute", func(t *testing.T) {
|
|
r = prepareGetByAttributeRequest(ctx, bktName, keyAttr, valAttr)
|
|
hc.Handler().DownloadByAttribute(r)
|
|
require.Equal(t, content, string(r.Response.Body()))
|
|
})
|
|
|
|
t.Run("head by attribute", func(t *testing.T) {
|
|
r = prepareGetByAttributeRequest(ctx, bktName, keyAttr, valAttr)
|
|
hc.Handler().HeadByAttribute(r)
|
|
require.Equal(t, putRes.ObjectID, string(r.Response.Header.Peek(hdrObjectID)))
|
|
require.Equal(t, putRes.ContainerID, string(r.Response.Header.Peek(hdrContainerID)))
|
|
})
|
|
|
|
t.Run("zip", func(t *testing.T) {
|
|
r = prepareGetZipped(ctx, bktName, "")
|
|
hc.Handler().DownloadZipped(r)
|
|
|
|
readerAt := bytes.NewReader(r.Response.Body())
|
|
zipReader, err := zip.NewReader(readerAt, int64(len(r.Response.Body())))
|
|
require.NoError(t, err)
|
|
require.Len(t, zipReader.File, 1)
|
|
require.Equal(t, objFileName, zipReader.File[0].Name)
|
|
f, err := zipReader.File[0].Open()
|
|
require.NoError(t, err)
|
|
defer func() {
|
|
inErr := f.Close()
|
|
require.NoError(t, inErr)
|
|
}()
|
|
data, err := io.ReadAll(f)
|
|
require.NoError(t, err)
|
|
require.Equal(t, content, string(data))
|
|
})
|
|
}
|
|
|
|
func TestFindObjectByAttribute(t *testing.T) {
|
|
hc, err := prepareHandlerContext()
|
|
require.NoError(t, err)
|
|
hc.cfg.additionalSearch = true
|
|
|
|
bktName := "bucket"
|
|
cnrID, cnr, err := hc.prepareContainer(bktName, acl.PublicRWExtended)
|
|
require.NoError(t, err)
|
|
hc.frostfs.SetContainer(cnrID, cnr)
|
|
|
|
ctx := context.Background()
|
|
ctx = middleware.SetNamespace(ctx, "")
|
|
|
|
content := "hello"
|
|
r, err := prepareUploadRequest(ctx, cnrID.EncodeToString(), content)
|
|
require.NoError(t, err)
|
|
|
|
hc.Handler().Upload(r)
|
|
require.Equal(t, r.Response.StatusCode(), http.StatusOK)
|
|
|
|
var putRes putResponse
|
|
err = json.Unmarshal(r.Response.Body(), &putRes)
|
|
require.NoError(t, err)
|
|
|
|
testAttrVal1 := "test-attr-val1"
|
|
testAttrVal2 := "test-attr-val2"
|
|
testAttrVal3 := "test-attr-val3"
|
|
|
|
for _, tc := range []struct {
|
|
name string
|
|
firstAttr object.Attribute
|
|
secondAttr object.Attribute
|
|
reqAttrKey string
|
|
reqAttrValue string
|
|
err string
|
|
additionalSearch bool
|
|
}{
|
|
{
|
|
name: "success search by FileName",
|
|
firstAttr: prepareObjectAttributes(attrFilePath, testAttrVal1),
|
|
secondAttr: prepareObjectAttributes(attrFileName, testAttrVal2),
|
|
reqAttrKey: attrFileName,
|
|
reqAttrValue: testAttrVal2,
|
|
additionalSearch: false,
|
|
},
|
|
{
|
|
name: "failed search by FileName",
|
|
firstAttr: prepareObjectAttributes(attrFilePath, testAttrVal1),
|
|
secondAttr: prepareObjectAttributes(attrFileName, testAttrVal2),
|
|
reqAttrKey: attrFileName,
|
|
reqAttrValue: testAttrVal3,
|
|
err: "not found",
|
|
additionalSearch: false,
|
|
},
|
|
{
|
|
name: "success search by FilePath (with additional search)",
|
|
firstAttr: prepareObjectAttributes(attrFilePath, testAttrVal1),
|
|
secondAttr: prepareObjectAttributes(attrFileName, testAttrVal2),
|
|
reqAttrKey: attrFilePath,
|
|
reqAttrValue: testAttrVal2,
|
|
additionalSearch: true,
|
|
},
|
|
{
|
|
name: "failed by FilePath (with additional search)",
|
|
firstAttr: prepareObjectAttributes(attrFilePath, testAttrVal1),
|
|
secondAttr: prepareObjectAttributes(attrFileName, testAttrVal2),
|
|
reqAttrKey: attrFilePath,
|
|
reqAttrValue: testAttrVal3,
|
|
err: "not found",
|
|
additionalSearch: true,
|
|
},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
obj := hc.frostfs.objects[putRes.ContainerID+"/"+putRes.ObjectID]
|
|
obj.SetAttributes(tc.firstAttr, tc.secondAttr)
|
|
hc.cfg.additionalSearch = tc.additionalSearch
|
|
|
|
objID, err := hc.Handler().findObjectByAttribute(ctx, hc.Handler().log, cnrID, tc.reqAttrKey, tc.reqAttrValue)
|
|
if tc.err != "" {
|
|
require.Error(t, err)
|
|
require.Contains(t, err.Error(), tc.err)
|
|
return
|
|
}
|
|
|
|
require.NoError(t, err)
|
|
require.Equal(t, putRes.ObjectID, objID.EncodeToString())
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestNeedSearchByFileName(t *testing.T) {
|
|
hc, err := prepareHandlerContext()
|
|
require.NoError(t, err)
|
|
|
|
for _, tc := range []struct {
|
|
name string
|
|
attrKey string
|
|
attrVal string
|
|
additionalSearch bool
|
|
expected bool
|
|
}{
|
|
{
|
|
name: "need search - not contains slash",
|
|
attrKey: attrFilePath,
|
|
attrVal: "cat.png",
|
|
additionalSearch: true,
|
|
expected: true,
|
|
},
|
|
{
|
|
name: "need search - single lead slash",
|
|
attrKey: attrFilePath,
|
|
attrVal: "/cat.png",
|
|
additionalSearch: true,
|
|
expected: true,
|
|
},
|
|
{
|
|
name: "don't need search - single slash but not lead",
|
|
attrKey: attrFilePath,
|
|
attrVal: "cats/cat.png",
|
|
additionalSearch: true,
|
|
expected: false,
|
|
},
|
|
{
|
|
name: "don't need search - more one slash",
|
|
attrKey: attrFilePath,
|
|
attrVal: "/cats/cat.png",
|
|
additionalSearch: true,
|
|
expected: false,
|
|
},
|
|
{
|
|
name: "don't need search - incorrect attribute key",
|
|
attrKey: attrFileName,
|
|
attrVal: "cat.png",
|
|
additionalSearch: true,
|
|
expected: false,
|
|
},
|
|
{
|
|
name: "don't need search - additional search disabled",
|
|
attrKey: attrFilePath,
|
|
attrVal: "cat.png",
|
|
additionalSearch: false,
|
|
expected: false,
|
|
},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
hc.cfg.additionalSearch = tc.additionalSearch
|
|
|
|
res := hc.h.needSearchByFileName(tc.attrKey, tc.attrVal)
|
|
require.Equal(t, tc.expected, res)
|
|
})
|
|
}
|
|
}
|
|
|
|
func prepareUploadRequest(ctx context.Context, bucket, content string) (*fasthttp.RequestCtx, error) {
|
|
r := new(fasthttp.RequestCtx)
|
|
utils.SetContextToRequest(ctx, r)
|
|
r.SetUserValue("cid", bucket)
|
|
return r, fillMultipartBody(r, content)
|
|
}
|
|
|
|
func prepareGetRequest(ctx context.Context, bucket, objID string) *fasthttp.RequestCtx {
|
|
r := new(fasthttp.RequestCtx)
|
|
utils.SetContextToRequest(ctx, r)
|
|
r.SetUserValue("cid", bucket)
|
|
r.SetUserValue("oid", objID)
|
|
return r
|
|
}
|
|
|
|
func prepareGetByAttributeRequest(ctx context.Context, bucket, attrKey, attrVal string) *fasthttp.RequestCtx {
|
|
r := new(fasthttp.RequestCtx)
|
|
utils.SetContextToRequest(ctx, r)
|
|
r.SetUserValue("cid", bucket)
|
|
r.SetUserValue("attr_key", attrKey)
|
|
r.SetUserValue("attr_val", attrVal)
|
|
return r
|
|
}
|
|
|
|
func prepareGetZipped(ctx context.Context, bucket, prefix string) *fasthttp.RequestCtx {
|
|
r := new(fasthttp.RequestCtx)
|
|
utils.SetContextToRequest(ctx, r)
|
|
r.SetUserValue("cid", bucket)
|
|
r.SetUserValue("prefix", prefix)
|
|
return r
|
|
}
|
|
|
|
func prepareObjectAttributes(attrKey, attrValue string) object.Attribute {
|
|
attr := object.NewAttribute()
|
|
attr.SetKey(attrKey)
|
|
attr.SetValue(attrValue)
|
|
return *attr
|
|
}
|
|
|
|
const (
|
|
keyAttr = "User-Attribute"
|
|
valAttr = "user value"
|
|
objFileName = "newFile.txt"
|
|
)
|
|
|
|
func fillMultipartBody(r *fasthttp.RequestCtx, content string) error {
|
|
attributes := map[string]string{
|
|
object.AttributeFileName: objFileName,
|
|
keyAttr: valAttr,
|
|
}
|
|
|
|
var buff bytes.Buffer
|
|
w := multipart.NewWriter(&buff)
|
|
fw, err := w.CreateFormFile("file", attributes[object.AttributeFileName])
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if _, err = io.Copy(fw, bytes.NewBufferString(content)); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err = w.Close(); err != nil {
|
|
return err
|
|
}
|
|
|
|
r.Request.SetBodyStream(&buff, buff.Len())
|
|
r.Request.Header.Set("Content-Type", w.FormDataContentType())
|
|
r.Request.Header.Set("X-Attribute-"+keyAttr, valAttr)
|
|
|
|
return nil
|
|
}
|