Nikita Zinkevich
5e4a68105d
Add tree service's GetBucketSettings to use them to check for protocol to use (S3 or native). Also add mock implementations for this and GetLatestVersion methods. Signed-off-by: Nikita Zinkevich <n.zinkevich@yadro.com>
414 lines
11 KiB
Go
414 lines
11 KiB
Go
package handler
|
|
|
|
import (
|
|
"archive/zip"
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"io"
|
|
"mime/multipart"
|
|
"net/http"
|
|
"sort"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl"
|
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
|
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
|
treepool "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
|
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
|
"github.com/panjf2000/ants/v2"
|
|
"github.com/stretchr/testify/require"
|
|
"github.com/valyala/fasthttp"
|
|
"go.uber.org/zap"
|
|
)
|
|
|
|
type treeServiceMock struct {
|
|
settings map[string]*data.BucketSettings
|
|
versions map[string]map[string][]*data.NodeVersion
|
|
system map[string]map[string]*data.BaseNodeVersion
|
|
}
|
|
|
|
func newTreeService() *treeServiceMock {
|
|
return &treeServiceMock{
|
|
settings: make(map[string]*data.BucketSettings),
|
|
versions: make(map[string]map[string][]*data.NodeVersion),
|
|
system: make(map[string]map[string]*data.BaseNodeVersion),
|
|
}
|
|
}
|
|
|
|
func (t *treeServiceMock) GetSubTreeByPrefix(context.Context, *data.BucketInfo, string, bool) ([]tree.NodeResponse, string, error) {
|
|
return nil, "", nil
|
|
}
|
|
|
|
func (t *treeServiceMock) GetSettingsNode(_ context.Context, bktInfo *data.BucketInfo) (*data.BucketSettings, error) {
|
|
settings, ok := t.settings[bktInfo.CID.EncodeToString()]
|
|
if !ok {
|
|
return nil, treepool.ErrNodeNotFound
|
|
}
|
|
|
|
return settings, nil
|
|
}
|
|
|
|
func (t *treeServiceMock) PutSettingsNode(_ context.Context, bktInfo *data.BucketInfo, settings *data.BucketSettings) error {
|
|
t.settings[bktInfo.CID.EncodeToString()] = settings
|
|
return nil
|
|
}
|
|
|
|
func (t *treeServiceMock) GetLatestVersion(_ context.Context, cnrID *cid.ID, objectName string) (*data.NodeVersion, error) {
|
|
cnrVersionsMap, ok := t.versions[cnrID.EncodeToString()]
|
|
if !ok {
|
|
return nil, treepool.ErrNodeNotFound
|
|
}
|
|
|
|
versions, ok := cnrVersionsMap[objectName]
|
|
if !ok {
|
|
return nil, treepool.ErrNodeNotFound
|
|
}
|
|
|
|
sort.Slice(versions, func(i, j int) bool {
|
|
return versions[i].ID < versions[j].ID
|
|
})
|
|
|
|
if len(versions) != 0 {
|
|
return versions[len(versions)-1], nil
|
|
}
|
|
|
|
return nil, treepool.ErrNodeNotFound
|
|
}
|
|
|
|
func (t *treeServiceMock) AddVersion(_ context.Context, bktInfo *data.BucketInfo, newVersion *data.NodeVersion) (uint64, error) {
|
|
cnrVersionsMap, ok := t.versions[bktInfo.CID.EncodeToString()]
|
|
if !ok {
|
|
t.versions[bktInfo.CID.EncodeToString()] = map[string][]*data.NodeVersion{
|
|
newVersion.FilePath: {newVersion},
|
|
}
|
|
return newVersion.ID, nil
|
|
}
|
|
|
|
versions, ok := cnrVersionsMap[newVersion.FilePath]
|
|
if !ok {
|
|
cnrVersionsMap[newVersion.FilePath] = []*data.NodeVersion{newVersion}
|
|
return newVersion.ID, nil
|
|
}
|
|
|
|
sort.Slice(versions, func(i, j int) bool {
|
|
return versions[i].ID < versions[j].ID
|
|
})
|
|
|
|
if len(versions) != 0 {
|
|
newVersion.ID = versions[len(versions)-1].ID + 1
|
|
}
|
|
|
|
result := versions
|
|
|
|
if newVersion.IsUnversioned {
|
|
result = make([]*data.NodeVersion, 0, len(versions))
|
|
for _, node := range versions {
|
|
if !node.IsUnversioned {
|
|
result = append(result, node)
|
|
}
|
|
}
|
|
}
|
|
|
|
cnrVersionsMap[newVersion.FilePath] = append(result, newVersion)
|
|
|
|
return newVersion.ID, nil
|
|
}
|
|
|
|
type configMock struct {
|
|
}
|
|
|
|
func (c *configMock) DefaultTimestamp() bool {
|
|
return false
|
|
}
|
|
|
|
func (c *configMock) ZipCompression() bool {
|
|
return false
|
|
}
|
|
|
|
func (c *configMock) IndexPageEnabled() bool {
|
|
return false
|
|
}
|
|
|
|
func (c *configMock) IndexPageTemplate() string {
|
|
return ""
|
|
}
|
|
|
|
func (c *configMock) IndexPageNativeTemplate() string {
|
|
return ""
|
|
}
|
|
|
|
func (c *configMock) ClientCut() bool {
|
|
return false
|
|
}
|
|
|
|
func (c *configMock) BufferMaxSizeForPut() uint64 {
|
|
return 0
|
|
}
|
|
|
|
func (c *configMock) NamespaceHeader() string {
|
|
return ""
|
|
}
|
|
|
|
type handlerContext struct {
|
|
key *keys.PrivateKey
|
|
owner user.ID
|
|
|
|
h *Handler
|
|
frostfs *TestFrostFS
|
|
tree *treeServiceMock
|
|
cfg *configMock
|
|
}
|
|
|
|
func (hc *handlerContext) Handler() *Handler {
|
|
return hc.h
|
|
}
|
|
|
|
func prepareHandlerContext() (*handlerContext, error) {
|
|
logger, err := zap.NewDevelopment()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
key, err := keys.NewPrivateKey()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
var owner user.ID
|
|
user.IDFromKey(&owner, key.PrivateKey.PublicKey)
|
|
|
|
testFrostFS := NewTestFrostFS(key)
|
|
|
|
testResolver := &resolver.Resolver{Name: "test_resolver"}
|
|
testResolver.SetResolveFunc(func(_ context.Context, name string) (*cid.ID, error) {
|
|
return testFrostFS.ContainerID(name)
|
|
})
|
|
|
|
params := &AppParams{
|
|
Logger: logger,
|
|
FrostFS: testFrostFS,
|
|
Owner: &owner,
|
|
Resolver: testResolver,
|
|
Cache: cache.NewBucketCache(&cache.Config{
|
|
Size: 1,
|
|
Lifetime: 1,
|
|
Logger: logger,
|
|
}),
|
|
}
|
|
|
|
treeMock := newTreeService()
|
|
cfgMock := &configMock{}
|
|
|
|
workerPool, err := ants.NewPool(1)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
handler := New(params, cfgMock, treeMock, workerPool)
|
|
|
|
return &handlerContext{
|
|
key: key,
|
|
owner: owner,
|
|
h: handler,
|
|
frostfs: testFrostFS,
|
|
tree: treeMock,
|
|
cfg: cfgMock,
|
|
}, nil
|
|
}
|
|
|
|
func (hc *handlerContext) prepareContainer(name string, basicACL acl.Basic) (cid.ID, *container.Container, error) {
|
|
var pp netmap.PlacementPolicy
|
|
err := pp.DecodeString("REP 1")
|
|
if err != nil {
|
|
return cid.ID{}, nil, err
|
|
}
|
|
|
|
var cnr container.Container
|
|
cnr.Init()
|
|
cnr.SetOwner(hc.owner)
|
|
cnr.SetPlacementPolicy(pp)
|
|
cnr.SetBasicACL(basicACL)
|
|
|
|
var domain container.Domain
|
|
domain.SetName(name)
|
|
container.WriteDomain(&cnr, domain)
|
|
container.SetName(&cnr, name)
|
|
container.SetCreationTime(&cnr, time.Now())
|
|
|
|
cnrID := cidtest.ID()
|
|
|
|
for op := acl.OpObjectGet; op < acl.OpObjectHash; op++ {
|
|
hc.frostfs.AllowUserOperation(cnrID, hc.owner, op, oid.ID{})
|
|
if basicACL.IsOpAllowed(op, acl.RoleOthers) {
|
|
hc.frostfs.AllowUserOperation(cnrID, user.ID{}, op, oid.ID{})
|
|
}
|
|
}
|
|
|
|
return cnrID, &cnr, nil
|
|
}
|
|
|
|
func TestBasic(t *testing.T) {
|
|
hc, err := prepareHandlerContext()
|
|
require.NoError(t, err)
|
|
|
|
bktName := "bucket"
|
|
cnrID, cnr, err := hc.prepareContainer(bktName, acl.PublicRWExtended)
|
|
require.NoError(t, err)
|
|
hc.frostfs.SetContainer(cnrID, cnr)
|
|
|
|
ctx := context.Background()
|
|
ctx = middleware.SetNamespace(ctx, "")
|
|
|
|
content := "hello"
|
|
r, err := prepareUploadRequest(ctx, cnrID.EncodeToString(), content)
|
|
require.NoError(t, err)
|
|
|
|
hc.Handler().Upload(r)
|
|
require.Equal(t, r.Response.StatusCode(), http.StatusOK)
|
|
|
|
var putRes putResponse
|
|
err = json.Unmarshal(r.Response.Body(), &putRes)
|
|
require.NoError(t, err)
|
|
|
|
obj := hc.frostfs.objects[putRes.ContainerID+"/"+putRes.ObjectID]
|
|
|
|
objID, ok := obj.ID()
|
|
require.True(t, ok)
|
|
|
|
_, err = hc.tree.AddVersion(context.TODO(), &data.BucketInfo{CID: cnrID}, &data.NodeVersion{
|
|
BaseNodeVersion: data.BaseNodeVersion{
|
|
OID: objID,
|
|
FilePath: objFileName,
|
|
},
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
attr := object.NewAttribute()
|
|
attr.SetKey(object.AttributeFilePath)
|
|
attr.SetValue(objFileName)
|
|
obj.SetAttributes(append(obj.Attributes(), *attr)...)
|
|
|
|
t.Run("get", func(t *testing.T) {
|
|
r = prepareGetRequest(ctx, cnrID.EncodeToString(), putRes.ObjectID)
|
|
hc.Handler().DownloadByAddressOrBucketName(r)
|
|
require.Equal(t, content, string(r.Response.Body()))
|
|
})
|
|
|
|
t.Run("head", func(t *testing.T) {
|
|
r = prepareGetRequest(ctx, cnrID.EncodeToString(), putRes.ObjectID)
|
|
hc.Handler().HeadByAddressOrBucketName(r)
|
|
require.Equal(t, putRes.ObjectID, string(r.Response.Header.Peek(hdrObjectID)))
|
|
require.Equal(t, putRes.ContainerID, string(r.Response.Header.Peek(hdrContainerID)))
|
|
})
|
|
|
|
t.Run("get by attribute", func(t *testing.T) {
|
|
r = prepareGetByAttributeRequest(ctx, bktName, keyAttr, valAttr)
|
|
hc.Handler().DownloadByAttribute(r)
|
|
require.Equal(t, content, string(r.Response.Body()))
|
|
})
|
|
|
|
t.Run("head by attribute", func(t *testing.T) {
|
|
r = prepareGetByAttributeRequest(ctx, bktName, keyAttr, valAttr)
|
|
hc.Handler().HeadByAttribute(r)
|
|
require.Equal(t, putRes.ObjectID, string(r.Response.Header.Peek(hdrObjectID)))
|
|
require.Equal(t, putRes.ContainerID, string(r.Response.Header.Peek(hdrContainerID)))
|
|
})
|
|
|
|
t.Run("zip", func(t *testing.T) {
|
|
r = prepareGetZipped(ctx, bktName, "")
|
|
hc.Handler().DownloadZipped(r)
|
|
|
|
readerAt := bytes.NewReader(r.Response.Body())
|
|
zipReader, err := zip.NewReader(readerAt, int64(len(r.Response.Body())))
|
|
require.NoError(t, err)
|
|
require.Len(t, zipReader.File, 1)
|
|
require.Equal(t, objFileName, zipReader.File[0].Name)
|
|
f, err := zipReader.File[0].Open()
|
|
require.NoError(t, err)
|
|
defer func() {
|
|
inErr := f.Close()
|
|
require.NoError(t, inErr)
|
|
}()
|
|
data, err := io.ReadAll(f)
|
|
require.NoError(t, err)
|
|
require.Equal(t, content, string(data))
|
|
})
|
|
}
|
|
|
|
func prepareUploadRequest(ctx context.Context, bucket, content string) (*fasthttp.RequestCtx, error) {
|
|
r := new(fasthttp.RequestCtx)
|
|
utils.SetContextToRequest(ctx, r)
|
|
r.SetUserValue("cid", bucket)
|
|
return r, fillMultipartBody(r, content)
|
|
}
|
|
|
|
func prepareGetRequest(ctx context.Context, bucket, objID string) *fasthttp.RequestCtx {
|
|
r := new(fasthttp.RequestCtx)
|
|
utils.SetContextToRequest(ctx, r)
|
|
r.SetUserValue("cid", bucket)
|
|
r.SetUserValue("oid", objID)
|
|
return r
|
|
}
|
|
|
|
func prepareGetByAttributeRequest(ctx context.Context, bucket, attrKey, attrVal string) *fasthttp.RequestCtx {
|
|
r := new(fasthttp.RequestCtx)
|
|
utils.SetContextToRequest(ctx, r)
|
|
r.SetUserValue("cid", bucket)
|
|
r.SetUserValue("attr_key", attrKey)
|
|
r.SetUserValue("attr_val", attrVal)
|
|
return r
|
|
}
|
|
|
|
func prepareGetZipped(ctx context.Context, bucket, prefix string) *fasthttp.RequestCtx {
|
|
r := new(fasthttp.RequestCtx)
|
|
utils.SetContextToRequest(ctx, r)
|
|
r.SetUserValue("cid", bucket)
|
|
r.SetUserValue("prefix", prefix)
|
|
return r
|
|
}
|
|
|
|
const (
|
|
keyAttr = "User-Attribute"
|
|
valAttr = "user value"
|
|
objFileName = "newFile.txt"
|
|
)
|
|
|
|
func fillMultipartBody(r *fasthttp.RequestCtx, content string) error {
|
|
attributes := map[string]string{
|
|
object.AttributeFileName: objFileName,
|
|
keyAttr: valAttr,
|
|
}
|
|
|
|
var buff bytes.Buffer
|
|
w := multipart.NewWriter(&buff)
|
|
fw, err := w.CreateFormFile("file", attributes[object.AttributeFileName])
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if _, err = io.Copy(fw, bytes.NewBufferString(content)); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err = w.Close(); err != nil {
|
|
return err
|
|
}
|
|
|
|
r.Request.SetBodyStream(&buff, buff.Len())
|
|
r.Request.Header.Set("Content-Type", w.FormDataContentType())
|
|
r.Request.Header.Set("X-Attribute-"+keyAttr, valAttr)
|
|
|
|
return nil
|
|
}
|