[#151] index page: Add browse via native protocol
Signed-off-by: Nikita Zinkevich <n.zinkevich@yadro.com>
This commit is contained in:
parent
524117a344
commit
3b26cf921c
4 changed files with 164 additions and 48 deletions
|
@ -23,12 +23,16 @@ import (
|
||||||
|
|
||||||
// DownloadByAddressOrBucketName handles download requests using simple cid/oid or bucketname/key format.
|
// DownloadByAddressOrBucketName handles download requests using simple cid/oid or bucketname/key format.
|
||||||
func (h *Handler) DownloadByAddressOrBucketName(c *fasthttp.RequestCtx) {
|
func (h *Handler) DownloadByAddressOrBucketName(c *fasthttp.RequestCtx) {
|
||||||
cnrIDStr, _ := c.UserValue("cid").(string)
|
oidURLParam := c.UserValue("oid").(string)
|
||||||
var cnrID cid.ID
|
downloadQueryParam := c.QueryArgs().GetBool("download")
|
||||||
if err := cnrID.DecodeString(cnrIDStr); err != nil {
|
|
||||||
h.byS3Path(c, h.receiveFile)
|
switch {
|
||||||
} else {
|
case isObjectID(oidURLParam):
|
||||||
h.byNativeAddress(c, h.receiveFile)
|
h.byNativeAddress(c, h.receiveFile)
|
||||||
|
case !isContainerRoot(oidURLParam) && (downloadQueryParam || !isDir(oidURLParam)):
|
||||||
|
h.byS3Path(c, h.receiveFile)
|
||||||
|
default:
|
||||||
|
h.browseIndex(c)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -208,16 +208,6 @@ func (h *Handler) byNativeAddress(c *fasthttp.RequestCtx, f func(context.Context
|
||||||
|
|
||||||
objID := new(oid.ID)
|
objID := new(oid.ID)
|
||||||
if err = objID.DecodeString(idObj); err != nil {
|
if err = objID.DecodeString(idObj); err != nil {
|
||||||
if h.config.IndexPageEnabled() {
|
|
||||||
c.SetStatusCode(fasthttp.StatusNotFound)
|
|
||||||
h.browseObjects(c, browseParams{
|
|
||||||
bucketInfo: bktInfo,
|
|
||||||
prefix: idObj,
|
|
||||||
listObjects: h.getDirObjectsNative,
|
|
||||||
isNative: true,
|
|
||||||
})
|
|
||||||
return
|
|
||||||
}
|
|
||||||
log.Error(logs.WrongObjectID, zap.Error(err))
|
log.Error(logs.WrongObjectID, zap.Error(err))
|
||||||
response.Error(c, "wrong object id", fasthttp.StatusBadRequest)
|
response.Error(c, "wrong object id", fasthttp.StatusBadRequest)
|
||||||
return
|
return
|
||||||
|
@ -233,7 +223,6 @@ func (h *Handler) byNativeAddress(c *fasthttp.RequestCtx, f func(context.Context
|
||||||
func (h *Handler) byS3Path(c *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) {
|
func (h *Handler) byS3Path(c *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) {
|
||||||
bucketname := c.UserValue("cid").(string)
|
bucketname := c.UserValue("cid").(string)
|
||||||
key := c.UserValue("oid").(string)
|
key := c.UserValue("oid").(string)
|
||||||
download := c.QueryArgs().GetBool("download")
|
|
||||||
|
|
||||||
ctx := utils.GetContextFromRequest(c)
|
ctx := utils.GetContextFromRequest(c)
|
||||||
reqLog := utils.GetReqLogOrDefault(ctx, h.log)
|
reqLog := utils.GetReqLogOrDefault(ctx, h.log)
|
||||||
|
@ -252,20 +241,6 @@ func (h *Handler) byS3Path(c *fasthttp.RequestCtx, f func(context.Context, reque
|
||||||
}
|
}
|
||||||
|
|
||||||
foundOid, err := h.tree.GetLatestVersion(ctx, &bktInfo.CID, unescapedKey)
|
foundOid, err := h.tree.GetLatestVersion(ctx, &bktInfo.CID, unescapedKey)
|
||||||
if h.config.IndexPageEnabled() && !download && string(c.Method()) != fasthttp.MethodHead {
|
|
||||||
if isDir(unescapedKey) || isContainerRoot(unescapedKey) {
|
|
||||||
if code := checkErrorType(err); code == fasthttp.StatusNotFound || code == fasthttp.StatusOK {
|
|
||||||
c.SetStatusCode(code)
|
|
||||||
h.browseObjects(c, browseParams{
|
|
||||||
bucketInfo: bktInfo,
|
|
||||||
prefix: unescapedKey,
|
|
||||||
listObjects: h.getDirObjectsS3,
|
|
||||||
isNative: false,
|
|
||||||
})
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if errors.Is(err, tree.ErrNodeAccessDenied) {
|
if errors.Is(err, tree.ErrNodeAccessDenied) {
|
||||||
response.Error(c, "Access Denied", fasthttp.StatusForbidden)
|
response.Error(c, "Access Denied", fasthttp.StatusForbidden)
|
||||||
|
@ -412,3 +387,54 @@ func (h *Handler) readContainer(ctx context.Context, cnrID cid.ID) (*data.Bucket
|
||||||
|
|
||||||
return bktInfo, err
|
return bktInfo, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (h *Handler) browseIndex(c *fasthttp.RequestCtx) {
|
||||||
|
if !h.config.IndexPageEnabled() {
|
||||||
|
c.SetStatusCode(fasthttp.StatusNotFound)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
cidURLParam := c.UserValue("cid").(string)
|
||||||
|
oidURLParam := c.UserValue("oid").(string)
|
||||||
|
|
||||||
|
ctx := utils.GetContextFromRequest(c)
|
||||||
|
reqLog := utils.GetReqLogOrDefault(ctx, h.log)
|
||||||
|
log := reqLog.With(zap.String("cid", cidURLParam), zap.String("oid", oidURLParam))
|
||||||
|
|
||||||
|
unescapedKey, err := url.QueryUnescape(oidURLParam)
|
||||||
|
if err != nil {
|
||||||
|
logAndSendBucketError(c, log, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
bktInfo, err := h.getBucketInfo(ctx, cidURLParam, log)
|
||||||
|
if err != nil {
|
||||||
|
logAndSendBucketError(c, log, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
c.SetStatusCode(fasthttp.StatusOK)
|
||||||
|
|
||||||
|
listFunc := h.getDirObjectsS3
|
||||||
|
isNativeList := false
|
||||||
|
|
||||||
|
err = h.tree.CheckSettingsNodeExist(ctx, bktInfo)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, tree.ErrNodeNotFound) {
|
||||||
|
// tree probe failed, try to use native
|
||||||
|
listFunc = h.getDirObjectsNative
|
||||||
|
isNativeList = true
|
||||||
|
} else {
|
||||||
|
logAndSendBucketError(c, log, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
c.SetStatusCode(fasthttp.StatusOK)
|
||||||
|
h.browseObjects(c, browseParams{
|
||||||
|
bucketInfo: bktInfo,
|
||||||
|
prefix: unescapedKey,
|
||||||
|
listObjects: listFunc,
|
||||||
|
isNative: isNativeList,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
|
@ -2,14 +2,12 @@ package handler
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/response"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/response"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
|
||||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
|
@ -47,19 +45,13 @@ func isDir(name string) bool {
|
||||||
return strings.HasSuffix(name, "/")
|
return strings.HasSuffix(name, "/")
|
||||||
}
|
}
|
||||||
|
|
||||||
func isContainerRoot(key string) bool {
|
func isObjectID(s string) bool {
|
||||||
return key == ""
|
var objID oid.ID
|
||||||
|
return objID.DecodeString(s) == nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func checkErrorType(err error) int {
|
func isContainerRoot(key string) bool {
|
||||||
switch {
|
return key == ""
|
||||||
case err == nil:
|
|
||||||
return fasthttp.StatusOK
|
|
||||||
case errors.Is(err, tree.ErrNodeAccessDenied):
|
|
||||||
return fasthttp.StatusForbidden
|
|
||||||
default:
|
|
||||||
return fasthttp.StatusNotFound
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func loadAttributes(attrs []object.Attribute) map[string]string {
|
func loadAttributes(attrs []object.Attribute) map[string]string {
|
||||||
|
|
100
tree/tree.go
100
tree/tree.go
|
@ -30,6 +30,11 @@ type (
|
||||||
Meta map[string]string
|
Meta map[string]string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
multiSystemNode struct {
|
||||||
|
// the first element is latest
|
||||||
|
nodes []*treeNode
|
||||||
|
}
|
||||||
|
|
||||||
GetNodesParams struct {
|
GetNodesParams struct {
|
||||||
CnrID cid.ID
|
CnrID cid.ID
|
||||||
BktInfo *data.BucketInfo
|
BktInfo *data.BucketInfo
|
||||||
|
@ -51,17 +56,18 @@ var (
|
||||||
|
|
||||||
const (
|
const (
|
||||||
FileNameKey = "FileName"
|
FileNameKey = "FileName"
|
||||||
)
|
settingsFileName = "bucket-settings"
|
||||||
|
|
||||||
const (
|
|
||||||
oidKV = "OID"
|
oidKV = "OID"
|
||||||
|
uploadIDKV = "UploadId"
|
||||||
|
sizeKV = "Size"
|
||||||
|
|
||||||
// keys for delete marker nodes.
|
// keys for delete marker nodes.
|
||||||
isDeleteMarkerKV = "IsDeleteMarker"
|
isDeleteMarkerKV = "IsDeleteMarker"
|
||||||
sizeKV = "Size"
|
|
||||||
|
|
||||||
// versionTree -- ID of a tree with object versions.
|
// versionTree -- ID of a tree with object versions.
|
||||||
versionTree = "version"
|
versionTree = "version"
|
||||||
|
systemTree = "system"
|
||||||
|
|
||||||
separator = "/"
|
separator = "/"
|
||||||
)
|
)
|
||||||
|
@ -135,6 +141,45 @@ func newNodeVersionFromTreeNode(treeNode *treeNode) *api.NodeVersion {
|
||||||
return version
|
return version
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func newMultiNode(nodes []NodeResponse) (*multiSystemNode, error) {
|
||||||
|
var (
|
||||||
|
err error
|
||||||
|
index int
|
||||||
|
maxTimestamp uint64
|
||||||
|
)
|
||||||
|
|
||||||
|
if len(nodes) == 0 {
|
||||||
|
return nil, errors.New("multi node must have at least one node")
|
||||||
|
}
|
||||||
|
|
||||||
|
treeNodes := make([]*treeNode, len(nodes))
|
||||||
|
|
||||||
|
for i, node := range nodes {
|
||||||
|
if treeNodes[i], err = newTreeNode(node); err != nil {
|
||||||
|
return nil, fmt.Errorf("parse system node response: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if timestamp := getMaxTimestamp(node); timestamp > maxTimestamp {
|
||||||
|
index = i
|
||||||
|
maxTimestamp = timestamp
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
treeNodes[0], treeNodes[index] = treeNodes[index], treeNodes[0]
|
||||||
|
|
||||||
|
return &multiSystemNode{
|
||||||
|
nodes: treeNodes,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *multiSystemNode) Latest() *treeNode {
|
||||||
|
return m.nodes[0]
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *multiSystemNode) Old() []*treeNode {
|
||||||
|
return m.nodes[1:]
|
||||||
|
}
|
||||||
|
|
||||||
func (c *Tree) GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*api.NodeVersion, error) {
|
func (c *Tree) GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*api.NodeVersion, error) {
|
||||||
nodes, err := c.GetVersions(ctx, cnrID, objectName)
|
nodes, err := c.GetVersions(ctx, cnrID, objectName)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -165,6 +210,55 @@ func (c *Tree) GetVersions(ctx context.Context, cnrID *cid.ID, objectName string
|
||||||
return c.service.GetNodes(ctx, p)
|
return c.service.GetNodes(ctx, p)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (c *Tree) CheckSettingsNodeExist(ctx context.Context, bktInfo *data.BucketInfo) error {
|
||||||
|
_, err := c.getSystemNode(ctx, bktInfo, settingsFileName)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Tree) getSystemNode(ctx context.Context, bktInfo *data.BucketInfo, name string) (*multiSystemNode, error) {
|
||||||
|
p := &GetNodesParams{
|
||||||
|
CnrID: bktInfo.CID,
|
||||||
|
BktInfo: bktInfo,
|
||||||
|
TreeID: systemTree,
|
||||||
|
Path: []string{name},
|
||||||
|
LatestOnly: false,
|
||||||
|
AllAttrs: true,
|
||||||
|
}
|
||||||
|
nodes, err := c.service.GetNodes(ctx, p)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
nodes = filterMultipartNodes(nodes)
|
||||||
|
|
||||||
|
if len(nodes) == 0 {
|
||||||
|
return nil, ErrNodeNotFound
|
||||||
|
}
|
||||||
|
|
||||||
|
return newMultiNode(nodes)
|
||||||
|
}
|
||||||
|
|
||||||
|
func filterMultipartNodes(nodes []NodeResponse) []NodeResponse {
|
||||||
|
res := make([]NodeResponse, 0, len(nodes))
|
||||||
|
|
||||||
|
LOOP:
|
||||||
|
for _, node := range nodes {
|
||||||
|
for _, meta := range node.GetMeta() {
|
||||||
|
if meta.GetKey() == uploadIDKV {
|
||||||
|
continue LOOP
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
res = append(res, node)
|
||||||
|
}
|
||||||
|
|
||||||
|
return res
|
||||||
|
}
|
||||||
|
|
||||||
func getLatestVersionNode(nodes []NodeResponse) (NodeResponse, error) {
|
func getLatestVersionNode(nodes []NodeResponse) (NodeResponse, error) {
|
||||||
var (
|
var (
|
||||||
maxCreationTime uint64
|
maxCreationTime uint64
|
||||||
|
|
Loading…
Reference in a new issue