[#233] Fix browsing
Some checks failed
/ DCO (pull_request) Successful in 30s
/ Vulncheck (pull_request) Successful in 53s
/ Builds (pull_request) Successful in 1m19s
/ OCI image (pull_request) Successful in 1m39s
/ Lint (pull_request) Failing after 1m35s
/ Tests (pull_request) Successful in 57s
/ Integration tests (pull_request) Successful in 6m5s
Some checks failed
/ DCO (pull_request) Successful in 30s
/ Vulncheck (pull_request) Successful in 53s
/ Builds (pull_request) Successful in 1m19s
/ OCI image (pull_request) Successful in 1m39s
/ Lint (pull_request) Failing after 1m35s
/ Tests (pull_request) Successful in 57s
/ Integration tests (pull_request) Successful in 6m5s
Simplify tree listing (we need only nodes in exactly the same parent level) Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
This commit is contained in:
parent
e579549b41
commit
9750521598
10 changed files with 302 additions and 114 deletions
|
@ -130,11 +130,15 @@ func parentDir(prefix string) string {
|
|||
return prefix[index:]
|
||||
}
|
||||
|
||||
func trimPrefix(encPrefix string) string {
|
||||
func getParent(encPrefix string) string {
|
||||
prefix, err := url.PathUnescape(encPrefix)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
if prefix != "" && prefix[len(prefix)-1] == '/' {
|
||||
prefix = prefix[:len(prefix)-1]
|
||||
}
|
||||
|
||||
slashIndex := strings.LastIndex(prefix, "/")
|
||||
if slashIndex == -1 {
|
||||
return ""
|
||||
|
@ -164,7 +168,11 @@ type GetObjectsResponse struct {
|
|||
}
|
||||
|
||||
func (h *Handler) getDirObjectsS3(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) (*GetObjectsResponse, error) {
|
||||
nodes, _, err := h.tree.GetSubTreeByPrefix(ctx, bucketInfo, prefix, true)
|
||||
if prefix != "" && prefix[len(prefix)-1] == '/' {
|
||||
prefix = prefix[:len(prefix)-1]
|
||||
}
|
||||
|
||||
nodes, err := h.tree.GetSubTreeByPrefix(ctx, bucketInfo, prefix, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
@ -185,7 +193,7 @@ func (h *Handler) getDirObjectsS3(ctx context.Context, bucketInfo *data.BucketIn
|
|||
if obj.IsDeleteMarker {
|
||||
continue
|
||||
}
|
||||
obj.FilePath = prefix + obj.FileName
|
||||
obj.FilePath = prefix + "/" + obj.FileName
|
||||
obj.GetURL = "/get/" + bucketInfo.Name + urlencode(obj.FilePath)
|
||||
result.objects = append(result.objects, obj)
|
||||
}
|
||||
|
@ -194,9 +202,9 @@ func (h *Handler) getDirObjectsS3(ctx context.Context, bucketInfo *data.BucketIn
|
|||
}
|
||||
|
||||
func (h *Handler) getDirObjectsNative(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) (*GetObjectsResponse, error) {
|
||||
var basePath string
|
||||
if ind := strings.LastIndex(prefix, "/"); ind != -1 {
|
||||
basePath = prefix[:ind+1]
|
||||
basePath := prefix
|
||||
if basePath != "" && basePath[len(basePath)-1] != '/' {
|
||||
basePath += "/"
|
||||
}
|
||||
|
||||
filters := object.NewSearchFilters()
|
||||
|
@ -342,7 +350,7 @@ func (h *Handler) browseObjects(ctx context.Context, req *fasthttp.RequestCtx, p
|
|||
|
||||
tmpl, err := template.New("index").Funcs(template.FuncMap{
|
||||
"formatSize": formatSize,
|
||||
"trimPrefix": trimPrefix,
|
||||
"getParent": getParent,
|
||||
"urlencode": urlencode,
|
||||
"parentDir": parentDir,
|
||||
}).Parse(h.config.IndexPageTemplate())
|
||||
|
@ -356,9 +364,14 @@ func (h *Handler) browseObjects(ctx context.Context, req *fasthttp.RequestCtx, p
|
|||
bucketName = p.bucketInfo.CID.EncodeToString()
|
||||
protocol = FrostfsProtocol
|
||||
}
|
||||
prefix := p.prefix
|
||||
if prefix != "" && prefix[len(prefix)-1] != '/' {
|
||||
prefix += "/"
|
||||
}
|
||||
|
||||
if err = tmpl.Execute(req, &BrowsePageData{
|
||||
Container: bucketName,
|
||||
Prefix: p.prefix,
|
||||
Prefix: prefix,
|
||||
Objects: objects,
|
||||
Protocol: protocol,
|
||||
HasErrors: p.objects.hasErrors,
|
||||
|
|
|
@ -14,8 +14,8 @@ import (
|
|||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
|
@ -51,7 +51,7 @@ func (h *Handler) DownloadByAddressOrBucketName(req *fasthttp.RequestCtx) {
|
|||
}
|
||||
|
||||
checkS3Err := h.tree.CheckSettingsNodeExists(ctx, bktInfo)
|
||||
if checkS3Err != nil && !errors.Is(checkS3Err, layer.ErrNodeNotFound) {
|
||||
if checkS3Err != nil && !errors.Is(checkS3Err, tree.ErrNodeNotFound) {
|
||||
h.logAndSendError(ctx, req, logs.FailedToCheckIfSettingsNodeExist, checkS3Err)
|
||||
return
|
||||
}
|
||||
|
@ -156,7 +156,7 @@ func (h *Handler) byS3PathMiddleware(handler func(context.Context, *fasthttp.Req
|
|||
return false
|
||||
}
|
||||
|
||||
if !errors.Is(err, layer.ErrNodeNotFound) {
|
||||
if !errors.Is(err, tree.ErrNodeNotFound) {
|
||||
h.logAndSendError(ctx, prm.Request, logs.FailedToGetLatestVersionOfIndexObject, err, zap.String("path", path))
|
||||
return false
|
||||
}
|
||||
|
|
|
@ -11,8 +11,8 @@ import (
|
|||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||
|
@ -173,7 +173,7 @@ type Handler struct {
|
|||
ownerID *user.ID
|
||||
config Config
|
||||
containerResolver ContainerResolver
|
||||
tree layer.TreeService
|
||||
tree *tree.Tree
|
||||
cache *cache.BucketCache
|
||||
workerPool *ants.Pool
|
||||
corsCnrID cid.ID
|
||||
|
@ -190,7 +190,7 @@ type AppParams struct {
|
|||
CORSCache *cache.CORSCache
|
||||
}
|
||||
|
||||
func New(params *AppParams, config Config, tree layer.TreeService, workerPool *ants.Pool) *Handler {
|
||||
func New(params *AppParams, config Config, tree *tree.Tree, workerPool *ants.Pool) *Handler {
|
||||
return &Handler{
|
||||
log: params.Logger,
|
||||
frostfs: params.FrostFS,
|
||||
|
|
|
@ -14,9 +14,10 @@ import (
|
|||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/templates"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
|
||||
v2container "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/container"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
|
||||
|
@ -36,32 +37,6 @@ import (
|
|||
"go.uber.org/zap/zaptest"
|
||||
)
|
||||
|
||||
type treeServiceMock struct {
|
||||
system map[string]map[string]*data.BaseNodeVersion
|
||||
}
|
||||
|
||||
func newTreeService() *treeServiceMock {
|
||||
return &treeServiceMock{
|
||||
system: make(map[string]map[string]*data.BaseNodeVersion),
|
||||
}
|
||||
}
|
||||
|
||||
func (t *treeServiceMock) CheckSettingsNodeExists(context.Context, *data.BucketInfo) error {
|
||||
_, ok := t.system["bucket-settings"]
|
||||
if !ok {
|
||||
return layer.ErrNodeNotFound
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *treeServiceMock) GetSubTreeByPrefix(context.Context, *data.BucketInfo, string, bool) ([]data.NodeInfo, string, error) {
|
||||
return nil, "", nil
|
||||
}
|
||||
|
||||
func (t *treeServiceMock) GetLatestVersion(context.Context, *cid.ID, string) (*data.NodeVersion, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
type configMock struct {
|
||||
additionalFilenameSearch bool
|
||||
additionalSlashSearch bool
|
||||
|
@ -82,7 +57,7 @@ func (c *configMock) IndexPageEnabled() bool {
|
|||
}
|
||||
|
||||
func (c *configMock) IndexPageTemplate() string {
|
||||
return ""
|
||||
return templates.DefaultIndexTemplate
|
||||
}
|
||||
|
||||
func (c *configMock) IndexPageNativeTemplate() string {
|
||||
|
@ -124,7 +99,7 @@ type handlerContext struct {
|
|||
|
||||
h *Handler
|
||||
frostfs *TestFrostFS
|
||||
tree *treeServiceMock
|
||||
tree *treeServiceClientMock
|
||||
cfg *configMock
|
||||
}
|
||||
|
||||
|
@ -174,14 +149,14 @@ func prepareHandlerContextBase(logger *zap.Logger) (*handlerContext, error) {
|
|||
}),
|
||||
}
|
||||
|
||||
treeMock := newTreeService()
|
||||
treeMock := newTreeServiceClientMock()
|
||||
cfgMock := &configMock{}
|
||||
|
||||
workerPool, err := ants.NewPool(1)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
handler := New(params, cfgMock, treeMock, workerPool)
|
||||
handler := New(params, cfgMock, tree.NewTree(treeMock, logger), workerPool)
|
||||
|
||||
return &handlerContext{
|
||||
key: key,
|
||||
|
@ -532,6 +507,100 @@ func TestGetObjectWithFallback(t *testing.T) {
|
|||
})
|
||||
}
|
||||
|
||||
func TestIndex(t *testing.T) {
|
||||
ctx := middleware.SetNamespace(context.Background(), "")
|
||||
|
||||
t.Run("s3", func(t *testing.T) {
|
||||
hc, cnrID := prepareHandlerAndBucket(t)
|
||||
|
||||
obj1ID := oidtest.ID()
|
||||
obj1 := object.New()
|
||||
obj1.SetID(obj1ID)
|
||||
obj1.SetPayload([]byte("obj1"))
|
||||
obj1.SetAttributes(prepareObjectAttributes(object.AttributeFilePath, "prefix/obj1"))
|
||||
hc.frostfs.objects[cnrID.String()+"/"+obj1ID.String()] = obj1
|
||||
|
||||
hc.tree.containers[cnrID.String()] = containerInfo{
|
||||
trees: map[string]map[string]nodeResponse{
|
||||
"system": {"bucket-settings": nodeResponse{nodeID: 1}},
|
||||
"version": {
|
||||
"": nodeResponse{}, //root
|
||||
"prefix": nodeResponse{
|
||||
nodeID: 1,
|
||||
meta: []nodeMeta{{key: tree.FileNameKey, value: []byte("prefix")}}},
|
||||
"obj1": nodeResponse{
|
||||
parentID: 1,
|
||||
nodeID: 2,
|
||||
meta: []nodeMeta{
|
||||
{key: tree.FileNameKey, value: []byte("obj1")},
|
||||
{key: "OID", value: []byte(obj1ID.String())},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
r := prepareGetRequest(ctx, cnrID.EncodeToString(), "prefix/")
|
||||
hc.Handler().DownloadByAddressOrBucketName(r)
|
||||
require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode())
|
||||
|
||||
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "prefix")
|
||||
hc.Handler().DownloadByAddressOrBucketName(r)
|
||||
require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode())
|
||||
|
||||
hc.cfg.indexEnabled = true
|
||||
|
||||
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "prefix")
|
||||
hc.Handler().DownloadByAddressOrBucketName(r)
|
||||
require.Contains(t, string(r.Response.Body()), "Index of s3://bucket/prefix")
|
||||
require.Contains(t, string(r.Response.Body()), obj1ID.String())
|
||||
|
||||
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "prefix/")
|
||||
hc.Handler().DownloadByAddressOrBucketName(r)
|
||||
require.Contains(t, string(r.Response.Body()), "Index of s3://bucket/prefix")
|
||||
require.Contains(t, string(r.Response.Body()), obj1ID.String())
|
||||
|
||||
r = prepareGetRequest(ctx, "bucket", "dummy")
|
||||
hc.Handler().DownloadByAddressOrBucketName(r)
|
||||
require.Contains(t, string(r.Response.Body()), "Index of s3://bucket/dummy")
|
||||
})
|
||||
|
||||
t.Run("native", func(t *testing.T) {
|
||||
hc, cnrID := prepareHandlerAndBucket(t)
|
||||
|
||||
obj1ID := oidtest.ID()
|
||||
obj1 := object.New()
|
||||
obj1.SetID(obj1ID)
|
||||
obj1.SetPayload([]byte("obj1"))
|
||||
obj1.SetAttributes(prepareObjectAttributes(object.AttributeFilePath, "prefix/obj1"))
|
||||
hc.frostfs.objects[cnrID.String()+"/"+obj1ID.String()] = obj1
|
||||
|
||||
r := prepareGetRequest(ctx, cnrID.EncodeToString(), "prefix/")
|
||||
hc.Handler().DownloadByAddressOrBucketName(r)
|
||||
require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode())
|
||||
|
||||
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "prefix")
|
||||
hc.Handler().DownloadByAddressOrBucketName(r)
|
||||
require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode())
|
||||
|
||||
hc.cfg.indexEnabled = true
|
||||
|
||||
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "prefix")
|
||||
hc.Handler().DownloadByAddressOrBucketName(r)
|
||||
require.Contains(t, string(r.Response.Body()), "Index of frostfs://"+cnrID.String()+"/prefix")
|
||||
require.Contains(t, string(r.Response.Body()), obj1ID.String())
|
||||
|
||||
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "prefix/")
|
||||
hc.Handler().DownloadByAddressOrBucketName(r)
|
||||
require.Contains(t, string(r.Response.Body()), "Index of frostfs://"+cnrID.String()+"/prefix")
|
||||
require.Contains(t, string(r.Response.Body()), obj1ID.String())
|
||||
|
||||
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "dummy")
|
||||
hc.Handler().DownloadByAddressOrBucketName(r)
|
||||
require.Contains(t, string(r.Response.Body()), "Index of frostfs://"+cnrID.String()+"/dummy")
|
||||
})
|
||||
}
|
||||
|
||||
func prepareUploadRequest(ctx context.Context, bucket, content string) (*fasthttp.RequestCtx, error) {
|
||||
r := new(fasthttp.RequestCtx)
|
||||
utils.SetContextToRequest(ctx, r)
|
||||
|
|
|
@ -9,8 +9,8 @@ import (
|
|||
"strconv"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
|
@ -142,7 +142,7 @@ func (h *Handler) HeadByAddressOrBucketName(req *fasthttp.RequestCtx) {
|
|||
}
|
||||
|
||||
checkS3Err := h.tree.CheckSettingsNodeExists(ctx, bktInfo)
|
||||
if checkS3Err != nil && !errors.Is(checkS3Err, layer.ErrNodeNotFound) {
|
||||
if checkS3Err != nil && !errors.Is(checkS3Err, tree.ErrNodeNotFound) {
|
||||
h.logAndSendError(ctx, req, logs.FailedToCheckIfSettingsNodeExist, checkS3Err)
|
||||
return
|
||||
}
|
||||
|
@ -157,7 +157,7 @@ func (h *Handler) HeadByAddressOrBucketName(req *fasthttp.RequestCtx) {
|
|||
indexPageEnabled := h.config.IndexPageEnabled()
|
||||
|
||||
if checkS3Err == nil {
|
||||
run(prm, h.errorMiddleware(logs.ObjectNotFound, layer.ErrNodeNotFound),
|
||||
run(prm, h.errorMiddleware(logs.ObjectNotFound, tree.ErrNodeNotFound),
|
||||
Middleware{Func: h.byS3PathMiddleware(h.headObject, noopFormer), Enabled: true},
|
||||
Middleware{Func: h.byS3PathMiddleware(h.headObject, indexFormer), Enabled: indexPageEnabled},
|
||||
)
|
||||
|
|
150
internal/handler/tree_service_client_mock_test.go
Normal file
150
internal/handler/tree_service_client_mock_test.go
Normal file
|
@ -0,0 +1,150 @@
|
|||
package handler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree"
|
||||
)
|
||||
|
||||
type nodeMeta struct {
|
||||
key string
|
||||
value []byte
|
||||
}
|
||||
|
||||
func (m nodeMeta) GetKey() string {
|
||||
return m.key
|
||||
}
|
||||
|
||||
func (m nodeMeta) GetValue() []byte {
|
||||
return m.value
|
||||
}
|
||||
|
||||
type nodeResponse struct {
|
||||
meta []nodeMeta
|
||||
nodeID uint64
|
||||
parentID uint64
|
||||
timestamp uint64
|
||||
}
|
||||
|
||||
func (n nodeResponse) GetNodeID() []uint64 {
|
||||
return []uint64{n.nodeID}
|
||||
}
|
||||
|
||||
func (n nodeResponse) GetParentID() []uint64 {
|
||||
return []uint64{n.parentID}
|
||||
}
|
||||
|
||||
func (n nodeResponse) GetTimestamp() []uint64 {
|
||||
return []uint64{n.timestamp}
|
||||
}
|
||||
|
||||
func (n nodeResponse) GetMeta() []tree.Meta {
|
||||
res := make([]tree.Meta, len(n.meta))
|
||||
for i, value := range n.meta {
|
||||
res[i] = value
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
func (n nodeResponse) getValue(key string) string {
|
||||
for _, value := range n.meta {
|
||||
if value.key == key {
|
||||
return string(value.value)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
type containerInfo struct {
|
||||
trees map[string]map[string]nodeResponse
|
||||
}
|
||||
|
||||
type treeServiceClientMock struct {
|
||||
containers map[string]containerInfo
|
||||
}
|
||||
|
||||
func newTreeServiceClientMock() *treeServiceClientMock {
|
||||
return &treeServiceClientMock{
|
||||
containers: make(map[string]containerInfo),
|
||||
}
|
||||
}
|
||||
|
||||
func (t *treeServiceClientMock) GetNodes(_ context.Context, p *tree.GetNodesParams) ([]tree.NodeResponse, error) {
|
||||
cnr, ok := t.containers[p.CnrID.EncodeToString()]
|
||||
if !ok {
|
||||
return nil, tree.ErrNodeNotFound
|
||||
}
|
||||
|
||||
tr, ok := cnr.trees[p.TreeID]
|
||||
if !ok {
|
||||
return nil, tree.ErrNodeNotFound
|
||||
}
|
||||
|
||||
node, ok := tr[strings.Join(p.Path, "/")]
|
||||
if !ok {
|
||||
return nil, tree.ErrNodeNotFound
|
||||
}
|
||||
|
||||
return []tree.NodeResponse{node}, nil
|
||||
}
|
||||
|
||||
func (t *treeServiceClientMock) GetSubTree(_ context.Context, bktInfo *data.BucketInfo, treeID string, rootID []uint64, depth uint32, _ bool) ([]tree.NodeResponse, error) {
|
||||
cnr, ok := t.containers[bktInfo.CID.EncodeToString()]
|
||||
if !ok {
|
||||
return nil, tree.ErrNodeNotFound
|
||||
}
|
||||
|
||||
tr, ok := cnr.trees[treeID]
|
||||
if !ok {
|
||||
return nil, tree.ErrNodeNotFound
|
||||
}
|
||||
|
||||
if len(rootID) != 1 {
|
||||
return nil, errors.New("invalid rootID")
|
||||
}
|
||||
|
||||
var root *nodeResponse
|
||||
for _, v := range tr {
|
||||
if v.nodeID == rootID[0] {
|
||||
root = &v
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if root == nil {
|
||||
return nil, tree.ErrNodeNotFound
|
||||
}
|
||||
|
||||
var res []nodeResponse
|
||||
if depth == 0 {
|
||||
for _, v := range tr {
|
||||
res = append(res, v)
|
||||
}
|
||||
} else {
|
||||
res = append(res, *root)
|
||||
depthIndex := 0
|
||||
for i := uint32(0); i < depth-1; i++ {
|
||||
childrenCount := 0
|
||||
for _, v := range tr {
|
||||
for j := range res[depthIndex:] {
|
||||
if v.parentID == res[j].nodeID {
|
||||
res = append(res, v)
|
||||
childrenCount++
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
depthIndex = len(res) - childrenCount
|
||||
}
|
||||
}
|
||||
|
||||
res2 := make([]tree.NodeResponse, len(res))
|
||||
for i := range res {
|
||||
res2[i] = res[i]
|
||||
}
|
||||
|
||||
return res2, nil
|
||||
}
|
|
@ -6,9 +6,9 @@ import (
|
|||
"fmt"
|
||||
"strings"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
|
@ -93,7 +93,7 @@ func formErrorResponse(err error) (string, int) {
|
|||
switch {
|
||||
case errors.Is(err, ErrAccessDenied):
|
||||
return fmt.Sprintf("Storage Access Denied:\n%v", err), fasthttp.StatusForbidden
|
||||
case errors.Is(err, layer.ErrNodeAccessDenied):
|
||||
case errors.Is(err, tree.ErrNodeAccessDenied):
|
||||
return fmt.Sprintf("Tree Access Denied:\n%v", err), fasthttp.StatusForbidden
|
||||
case errors.Is(err, ErrQuotaLimitReached):
|
||||
return fmt.Sprintf("Quota Reached:\n%v", err), fasthttp.StatusConflict
|
||||
|
@ -101,7 +101,7 @@ func formErrorResponse(err error) (string, int) {
|
|||
return fmt.Sprintf("Container Not Found:\n%v", err), fasthttp.StatusNotFound
|
||||
case errors.Is(err, ErrObjectNotFound):
|
||||
return fmt.Sprintf("Object Not Found:\n%v", err), fasthttp.StatusNotFound
|
||||
case errors.Is(err, layer.ErrNodeNotFound):
|
||||
case errors.Is(err, tree.ErrNodeNotFound):
|
||||
return fmt.Sprintf("Tree Node Not Found:\n%v", err), fasthttp.StatusNotFound
|
||||
case errors.Is(err, ErrGatewayTimeout):
|
||||
return fmt.Sprintf("Gateway Timeout:\n%v", err), fasthttp.StatusGatewayTimeout
|
||||
|
|
|
@ -1,24 +0,0 @@
|
|||
package layer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
)
|
||||
|
||||
// TreeService provide interface to interact with tree service using s3 data models.
|
||||
type TreeService interface {
|
||||
GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*data.NodeVersion, error)
|
||||
GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string, latestOnly bool) ([]data.NodeInfo, string, error)
|
||||
CheckSettingsNodeExists(ctx context.Context, bktInfo *data.BucketInfo) error
|
||||
}
|
||||
|
||||
var (
|
||||
// ErrNodeNotFound is returned from Tree service in case of not found error.
|
||||
ErrNodeNotFound = errors.New("not found")
|
||||
|
||||
// ErrNodeAccessDenied is returned from Tree service in case of access denied error.
|
||||
ErrNodeAccessDenied = errors.New("access denied")
|
||||
)
|
|
@ -1,11 +1,9 @@
|
|||
{{$container := .Container}}
|
||||
{{ $prefix := trimPrefix .Prefix }}
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8"/>
|
||||
<title>Index of {{.Protocol}}://{{$container}}
|
||||
/{{if $prefix}}/{{$prefix}}/{{end}}</title>
|
||||
<title>Index of {{.Protocol}}://{{$container}}/{{.Prefix}}</title>
|
||||
<style>
|
||||
.alert {
|
||||
width: 80%;
|
||||
|
@ -40,7 +38,7 @@
|
|||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Index of {{.Protocol}}://{{$container}}/{{if $prefix}}{{$prefix}}/{{end}}</h1>
|
||||
<h1>Index of {{.Protocol}}://{{$container}}/{{.Prefix}}</h1>
|
||||
{{ if .HasErrors }}
|
||||
<div class="alert">
|
||||
Errors occurred while processing the request. Perhaps some objects are missing
|
||||
|
@ -57,11 +55,11 @@
|
|||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{{ $trimmedPrefix := trimPrefix $prefix }}
|
||||
{{if $trimmedPrefix }}
|
||||
{{ $parentPrefix := getParent .Prefix }}
|
||||
{{if $parentPrefix }}
|
||||
<tr>
|
||||
<td>
|
||||
⮐<a href="/get/{{$container}}{{ urlencode $trimmedPrefix }}/">..</a>
|
||||
⮐<a href="/get/{{$container}}{{ urlencode $parentPrefix }}/">..</a>
|
||||
</td>
|
||||
<td></td>
|
||||
<td></td>
|
||||
|
|
46
tree/tree.go
46
tree/tree.go
|
@ -7,7 +7,6 @@ import (
|
|||
"strings"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
|
||||
|
@ -52,10 +51,10 @@ type (
|
|||
|
||||
var (
|
||||
// ErrNodeNotFound is returned from ServiceClient in case of not found error.
|
||||
ErrNodeNotFound = layer.ErrNodeNotFound
|
||||
ErrNodeNotFound = errors.New("not found")
|
||||
|
||||
// ErrNodeAccessDenied is returned from ServiceClient service in case of access denied error.
|
||||
ErrNodeAccessDenied = layer.ErrNodeAccessDenied
|
||||
ErrNodeAccessDenied = errors.New("access denied")
|
||||
)
|
||||
|
||||
const (
|
||||
|
@ -259,7 +258,7 @@ func (c *Tree) getSystemNode(ctx context.Context, bktInfo *data.BucketInfo, name
|
|||
nodes = filterMultipartNodes(nodes)
|
||||
|
||||
if len(nodes) == 0 {
|
||||
return nil, layer.ErrNodeNotFound
|
||||
return nil, ErrNodeNotFound
|
||||
}
|
||||
if len(nodes) != 1 {
|
||||
c.reqLogger(ctx).Warn(logs.FoundSeveralSystemTreeNodes, zap.String("name", name), logs.TagField(logs.TagExternalStorageTree))
|
||||
|
@ -303,7 +302,7 @@ func getLatestVersionNode(nodes []NodeResponse) (NodeResponse, error) {
|
|||
}
|
||||
|
||||
if targetIndexNode == -1 {
|
||||
return nil, fmt.Errorf("latest version: %w", layer.ErrNodeNotFound)
|
||||
return nil, fmt.Errorf("latest version: %w", ErrNodeNotFound)
|
||||
}
|
||||
|
||||
return nodes[targetIndexNode], nil
|
||||
|
@ -324,20 +323,23 @@ func pathFromName(objectName string) []string {
|
|||
return strings.Split(objectName, separator)
|
||||
}
|
||||
|
||||
func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string, latestOnly bool) ([]data.NodeInfo, string, error) {
|
||||
func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string, latestOnly bool) ([]data.NodeInfo, error) {
|
||||
ctx, span := tracing.StartSpanFromContext(ctx, "tree.GetSubTreeByPrefix")
|
||||
defer span.End()
|
||||
|
||||
rootID, tailPrefix, err := c.determinePrefixNode(ctx, bktInfo, versionTree, prefix)
|
||||
rootID, err := c.getPrefixNodeID(ctx, bktInfo, versionTree, strings.Split(prefix, separator))
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
if errors.Is(err, ErrNodeNotFound) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
subTree, err := c.service.GetSubTree(ctx, bktInfo, versionTree, rootID, 2, false)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrNodeNotFound) {
|
||||
return nil, "", nil
|
||||
return nil, nil
|
||||
}
|
||||
return nil, "", err
|
||||
return nil, err
|
||||
}
|
||||
|
||||
nodesMap := make(map[string][]NodeResponse, len(subTree))
|
||||
|
@ -347,10 +349,6 @@ func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo,
|
|||
}
|
||||
|
||||
fileName := GetFilename(node)
|
||||
if !strings.HasPrefix(fileName, tailPrefix) {
|
||||
continue
|
||||
}
|
||||
|
||||
nodes := nodesMap[fileName]
|
||||
|
||||
// Add all nodes if flag latestOnly is false.
|
||||
|
@ -374,7 +372,7 @@ func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo,
|
|||
result = append(result, nodeResponseToNodeInfo(nodes)...)
|
||||
}
|
||||
|
||||
return result, strings.TrimSuffix(prefix, tailPrefix), nil
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func nodeResponseToNodeInfo(nodes []NodeResponse) []data.NodeInfo {
|
||||
|
@ -386,22 +384,6 @@ func nodeResponseToNodeInfo(nodes []NodeResponse) []data.NodeInfo {
|
|||
return nodesInfo
|
||||
}
|
||||
|
||||
func (c *Tree) determinePrefixNode(ctx context.Context, bktInfo *data.BucketInfo, treeID, prefix string) ([]uint64, string, error) {
|
||||
rootID := []uint64{0}
|
||||
path := strings.Split(prefix, separator)
|
||||
tailPrefix := path[len(path)-1]
|
||||
|
||||
if len(path) > 1 {
|
||||
var err error
|
||||
rootID, err = c.getPrefixNodeID(ctx, bktInfo, treeID, path[:len(path)-1])
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
}
|
||||
|
||||
return rootID, tailPrefix, nil
|
||||
}
|
||||
|
||||
func (c *Tree) getPrefixNodeID(ctx context.Context, bktInfo *data.BucketInfo, treeID string, prefixPath []string) ([]uint64, error) {
|
||||
p := &GetNodesParams{
|
||||
CnrID: bktInfo.CID,
|
||||
|
@ -424,7 +406,7 @@ func (c *Tree) getPrefixNodeID(ctx context.Context, bktInfo *data.BucketInfo, tr
|
|||
}
|
||||
|
||||
if len(intermediateNodes) == 0 {
|
||||
return nil, layer.ErrNodeNotFound
|
||||
return nil, ErrNodeNotFound
|
||||
}
|
||||
|
||||
return intermediateNodes, nil
|
||||
|
|
Loading…
Add table
Reference in a new issue