diff --git a/internal/handler/browse.go b/internal/handler/browse.go index e1fc59d..d9e6625 100644 --- a/internal/handler/browse.go +++ b/internal/handler/browse.go @@ -130,11 +130,15 @@ func parentDir(prefix string) string { return prefix[index:] } -func trimPrefix(encPrefix string) string { +func getParent(encPrefix string) string { prefix, err := url.PathUnescape(encPrefix) if err != nil { return "" } + if prefix != "" && prefix[len(prefix)-1] == '/' { + prefix = prefix[:len(prefix)-1] + } + slashIndex := strings.LastIndex(prefix, "/") if slashIndex == -1 { return "" @@ -164,7 +168,11 @@ type GetObjectsResponse struct { } func (h *Handler) getDirObjectsS3(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) (*GetObjectsResponse, error) { - nodes, _, err := h.tree.GetSubTreeByPrefix(ctx, bucketInfo, prefix, true) + if prefix != "" && prefix[len(prefix)-1] == '/' { + prefix = prefix[:len(prefix)-1] + } + + nodes, err := h.tree.GetSubTreeByPrefix(ctx, bucketInfo, prefix, true) if err != nil { return nil, err } @@ -185,7 +193,7 @@ func (h *Handler) getDirObjectsS3(ctx context.Context, bucketInfo *data.BucketIn if obj.IsDeleteMarker { continue } - obj.FilePath = prefix + obj.FileName + obj.FilePath = prefix + "/" + obj.FileName obj.GetURL = "/get/" + bucketInfo.Name + urlencode(obj.FilePath) result.objects = append(result.objects, obj) } @@ -194,9 +202,9 @@ func (h *Handler) getDirObjectsS3(ctx context.Context, bucketInfo *data.BucketIn } func (h *Handler) getDirObjectsNative(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) (*GetObjectsResponse, error) { - var basePath string - if ind := strings.LastIndex(prefix, "/"); ind != -1 { - basePath = prefix[:ind+1] + basePath := prefix + if basePath != "" && basePath[len(basePath)-1] != '/' { + basePath += "/" } filters := object.NewSearchFilters() @@ -342,7 +350,7 @@ func (h *Handler) browseObjects(ctx context.Context, req *fasthttp.RequestCtx, p tmpl, err := template.New("index").Funcs(template.FuncMap{ "formatSize": formatSize, - "trimPrefix": trimPrefix, + "getParent": getParent, "urlencode": urlencode, "parentDir": parentDir, }).Parse(h.config.IndexPageTemplate()) @@ -356,9 +364,14 @@ func (h *Handler) browseObjects(ctx context.Context, req *fasthttp.RequestCtx, p bucketName = p.bucketInfo.CID.EncodeToString() protocol = FrostfsProtocol } + prefix := p.prefix + if prefix != "" && prefix[len(prefix)-1] != '/' { + prefix += "/" + } + if err = tmpl.Execute(req, &BrowsePageData{ Container: bucketName, - Prefix: p.prefix, + Prefix: prefix, Objects: objects, Protocol: protocol, HasErrors: p.objects.hasErrors, diff --git a/internal/handler/download.go b/internal/handler/download.go index 301d10f..b610082 100644 --- a/internal/handler/download.go +++ b/internal/handler/download.go @@ -14,8 +14,8 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -51,7 +51,7 @@ func (h *Handler) DownloadByAddressOrBucketName(req *fasthttp.RequestCtx) { } checkS3Err := h.tree.CheckSettingsNodeExists(ctx, bktInfo) - if checkS3Err != nil && !errors.Is(checkS3Err, layer.ErrNodeNotFound) { + if checkS3Err != nil && !errors.Is(checkS3Err, tree.ErrNodeNotFound) { h.logAndSendError(ctx, req, logs.FailedToCheckIfSettingsNodeExist, checkS3Err) return } @@ -156,7 +156,7 @@ func (h *Handler) byS3PathMiddleware(handler func(context.Context, *fasthttp.Req return false } - if !errors.Is(err, layer.ErrNodeNotFound) { + if !errors.Is(err, tree.ErrNodeNotFound) { h.logAndSendError(ctx, prm.Request, logs.FailedToGetLatestVersionOfIndexObject, err, zap.String("path", path)) return false } diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 59a19ed..e8a1faa 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -11,8 +11,8 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" @@ -173,7 +173,7 @@ type Handler struct { ownerID *user.ID config Config containerResolver ContainerResolver - tree layer.TreeService + tree *tree.Tree cache *cache.BucketCache workerPool *ants.Pool corsCnrID cid.ID @@ -190,7 +190,7 @@ type AppParams struct { CORSCache *cache.CORSCache } -func New(params *AppParams, config Config, tree layer.TreeService, workerPool *ants.Pool) *Handler { +func New(params *AppParams, config Config, tree *tree.Tree, workerPool *ants.Pool) *Handler { return &Handler{ log: params.Logger, frostfs: params.FrostFS, diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index dbb037d..622940e 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -14,9 +14,10 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/templates" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" v2container "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" @@ -36,32 +37,6 @@ import ( "go.uber.org/zap/zaptest" ) -type treeServiceMock struct { - system map[string]map[string]*data.BaseNodeVersion -} - -func newTreeService() *treeServiceMock { - return &treeServiceMock{ - system: make(map[string]map[string]*data.BaseNodeVersion), - } -} - -func (t *treeServiceMock) CheckSettingsNodeExists(context.Context, *data.BucketInfo) error { - _, ok := t.system["bucket-settings"] - if !ok { - return layer.ErrNodeNotFound - } - return nil -} - -func (t *treeServiceMock) GetSubTreeByPrefix(context.Context, *data.BucketInfo, string, bool) ([]data.NodeInfo, string, error) { - return nil, "", nil -} - -func (t *treeServiceMock) GetLatestVersion(context.Context, *cid.ID, string) (*data.NodeVersion, error) { - return nil, nil -} - type configMock struct { additionalFilenameSearch bool additionalSlashSearch bool @@ -82,7 +57,7 @@ func (c *configMock) IndexPageEnabled() bool { } func (c *configMock) IndexPageTemplate() string { - return "" + return templates.DefaultIndexTemplate } func (c *configMock) IndexPageNativeTemplate() string { @@ -124,7 +99,7 @@ type handlerContext struct { h *Handler frostfs *TestFrostFS - tree *treeServiceMock + tree *treeServiceClientMock cfg *configMock } @@ -174,14 +149,14 @@ func prepareHandlerContextBase(logger *zap.Logger) (*handlerContext, error) { }), } - treeMock := newTreeService() + treeMock := newTreeServiceClientMock() cfgMock := &configMock{} workerPool, err := ants.NewPool(1) if err != nil { return nil, err } - handler := New(params, cfgMock, treeMock, workerPool) + handler := New(params, cfgMock, tree.NewTree(treeMock, logger), workerPool) return &handlerContext{ key: key, @@ -532,6 +507,100 @@ func TestGetObjectWithFallback(t *testing.T) { }) } +func TestIndex(t *testing.T) { + ctx := middleware.SetNamespace(context.Background(), "") + + t.Run("s3", func(t *testing.T) { + hc, cnrID := prepareHandlerAndBucket(t) + + obj1ID := oidtest.ID() + obj1 := object.New() + obj1.SetID(obj1ID) + obj1.SetPayload([]byte("obj1")) + obj1.SetAttributes(prepareObjectAttributes(object.AttributeFilePath, "prefix/obj1")) + hc.frostfs.objects[cnrID.String()+"/"+obj1ID.String()] = obj1 + + hc.tree.containers[cnrID.String()] = containerInfo{ + trees: map[string]map[string]nodeResponse{ + "system": {"bucket-settings": nodeResponse{nodeID: 1}}, + "version": { + "": nodeResponse{}, //root + "prefix": nodeResponse{ + nodeID: 1, + meta: []nodeMeta{{key: tree.FileNameKey, value: []byte("prefix")}}}, + "obj1": nodeResponse{ + parentID: 1, + nodeID: 2, + meta: []nodeMeta{ + {key: tree.FileNameKey, value: []byte("obj1")}, + {key: "OID", value: []byte(obj1ID.String())}, + }, + }, + }, + }, + } + + r := prepareGetRequest(ctx, cnrID.EncodeToString(), "prefix/") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode()) + + r = prepareGetRequest(ctx, cnrID.EncodeToString(), "prefix") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode()) + + hc.cfg.indexEnabled = true + + r = prepareGetRequest(ctx, cnrID.EncodeToString(), "prefix") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Contains(t, string(r.Response.Body()), "Index of s3://bucket/prefix") + require.Contains(t, string(r.Response.Body()), obj1ID.String()) + + r = prepareGetRequest(ctx, cnrID.EncodeToString(), "prefix/") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Contains(t, string(r.Response.Body()), "Index of s3://bucket/prefix") + require.Contains(t, string(r.Response.Body()), obj1ID.String()) + + r = prepareGetRequest(ctx, "bucket", "dummy") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Contains(t, string(r.Response.Body()), "Index of s3://bucket/dummy") + }) + + t.Run("native", func(t *testing.T) { + hc, cnrID := prepareHandlerAndBucket(t) + + obj1ID := oidtest.ID() + obj1 := object.New() + obj1.SetID(obj1ID) + obj1.SetPayload([]byte("obj1")) + obj1.SetAttributes(prepareObjectAttributes(object.AttributeFilePath, "prefix/obj1")) + hc.frostfs.objects[cnrID.String()+"/"+obj1ID.String()] = obj1 + + r := prepareGetRequest(ctx, cnrID.EncodeToString(), "prefix/") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode()) + + r = prepareGetRequest(ctx, cnrID.EncodeToString(), "prefix") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode()) + + hc.cfg.indexEnabled = true + + r = prepareGetRequest(ctx, cnrID.EncodeToString(), "prefix") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Contains(t, string(r.Response.Body()), "Index of frostfs://"+cnrID.String()+"/prefix") + require.Contains(t, string(r.Response.Body()), obj1ID.String()) + + r = prepareGetRequest(ctx, cnrID.EncodeToString(), "prefix/") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Contains(t, string(r.Response.Body()), "Index of frostfs://"+cnrID.String()+"/prefix") + require.Contains(t, string(r.Response.Body()), obj1ID.String()) + + r = prepareGetRequest(ctx, cnrID.EncodeToString(), "dummy") + hc.Handler().DownloadByAddressOrBucketName(r) + require.Contains(t, string(r.Response.Body()), "Index of frostfs://"+cnrID.String()+"/dummy") + }) +} + func prepareUploadRequest(ctx context.Context, bucket, content string) (*fasthttp.RequestCtx, error) { r := new(fasthttp.RequestCtx) utils.SetContextToRequest(ctx, r) diff --git a/internal/handler/head.go b/internal/handler/head.go index e6d9a30..508dc37 100644 --- a/internal/handler/head.go +++ b/internal/handler/head.go @@ -9,8 +9,8 @@ import ( "strconv" "time" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -142,7 +142,7 @@ func (h *Handler) HeadByAddressOrBucketName(req *fasthttp.RequestCtx) { } checkS3Err := h.tree.CheckSettingsNodeExists(ctx, bktInfo) - if checkS3Err != nil && !errors.Is(checkS3Err, layer.ErrNodeNotFound) { + if checkS3Err != nil && !errors.Is(checkS3Err, tree.ErrNodeNotFound) { h.logAndSendError(ctx, req, logs.FailedToCheckIfSettingsNodeExist, checkS3Err) return } @@ -157,7 +157,7 @@ func (h *Handler) HeadByAddressOrBucketName(req *fasthttp.RequestCtx) { indexPageEnabled := h.config.IndexPageEnabled() if checkS3Err == nil { - run(prm, h.errorMiddleware(logs.ObjectNotFound, layer.ErrNodeNotFound), + run(prm, h.errorMiddleware(logs.ObjectNotFound, tree.ErrNodeNotFound), Middleware{Func: h.byS3PathMiddleware(h.headObject, noopFormer), Enabled: true}, Middleware{Func: h.byS3PathMiddleware(h.headObject, indexFormer), Enabled: indexPageEnabled}, ) diff --git a/internal/handler/tree_service_client_mock_test.go b/internal/handler/tree_service_client_mock_test.go new file mode 100644 index 0000000..d3f1ca5 --- /dev/null +++ b/internal/handler/tree_service_client_mock_test.go @@ -0,0 +1,150 @@ +package handler + +import ( + "context" + "errors" + "strings" + + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" +) + +type nodeMeta struct { + key string + value []byte +} + +func (m nodeMeta) GetKey() string { + return m.key +} + +func (m nodeMeta) GetValue() []byte { + return m.value +} + +type nodeResponse struct { + meta []nodeMeta + nodeID uint64 + parentID uint64 + timestamp uint64 +} + +func (n nodeResponse) GetNodeID() []uint64 { + return []uint64{n.nodeID} +} + +func (n nodeResponse) GetParentID() []uint64 { + return []uint64{n.parentID} +} + +func (n nodeResponse) GetTimestamp() []uint64 { + return []uint64{n.timestamp} +} + +func (n nodeResponse) GetMeta() []tree.Meta { + res := make([]tree.Meta, len(n.meta)) + for i, value := range n.meta { + res[i] = value + } + return res +} + +func (n nodeResponse) getValue(key string) string { + for _, value := range n.meta { + if value.key == key { + return string(value.value) + } + } + return "" +} + +type containerInfo struct { + trees map[string]map[string]nodeResponse +} + +type treeServiceClientMock struct { + containers map[string]containerInfo +} + +func newTreeServiceClientMock() *treeServiceClientMock { + return &treeServiceClientMock{ + containers: make(map[string]containerInfo), + } +} + +func (t *treeServiceClientMock) GetNodes(_ context.Context, p *tree.GetNodesParams) ([]tree.NodeResponse, error) { + cnr, ok := t.containers[p.CnrID.EncodeToString()] + if !ok { + return nil, tree.ErrNodeNotFound + } + + tr, ok := cnr.trees[p.TreeID] + if !ok { + return nil, tree.ErrNodeNotFound + } + + node, ok := tr[strings.Join(p.Path, "/")] + if !ok { + return nil, tree.ErrNodeNotFound + } + + return []tree.NodeResponse{node}, nil +} + +func (t *treeServiceClientMock) GetSubTree(_ context.Context, bktInfo *data.BucketInfo, treeID string, rootID []uint64, depth uint32, _ bool) ([]tree.NodeResponse, error) { + cnr, ok := t.containers[bktInfo.CID.EncodeToString()] + if !ok { + return nil, tree.ErrNodeNotFound + } + + tr, ok := cnr.trees[treeID] + if !ok { + return nil, tree.ErrNodeNotFound + } + + if len(rootID) != 1 { + return nil, errors.New("invalid rootID") + } + + var root *nodeResponse + for _, v := range tr { + if v.nodeID == rootID[0] { + root = &v + break + } + } + + if root == nil { + return nil, tree.ErrNodeNotFound + } + + var res []nodeResponse + if depth == 0 { + for _, v := range tr { + res = append(res, v) + } + } else { + res = append(res, *root) + depthIndex := 0 + for i := uint32(0); i < depth-1; i++ { + childrenCount := 0 + for _, v := range tr { + for j := range res[depthIndex:] { + if v.parentID == res[j].nodeID { + res = append(res, v) + childrenCount++ + break + } + } + } + depthIndex = len(res) - childrenCount + } + } + + res2 := make([]tree.NodeResponse, len(res)) + for i := range res { + res2[i] = res[i] + } + + return res2, nil +} diff --git a/internal/handler/utils.go b/internal/handler/utils.go index 8cb070d..c17b878 100644 --- a/internal/handler/utils.go +++ b/internal/handler/utils.go @@ -6,9 +6,9 @@ import ( "fmt" "strings" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -93,7 +93,7 @@ func formErrorResponse(err error) (string, int) { switch { case errors.Is(err, ErrAccessDenied): return fmt.Sprintf("Storage Access Denied:\n%v", err), fasthttp.StatusForbidden - case errors.Is(err, layer.ErrNodeAccessDenied): + case errors.Is(err, tree.ErrNodeAccessDenied): return fmt.Sprintf("Tree Access Denied:\n%v", err), fasthttp.StatusForbidden case errors.Is(err, ErrQuotaLimitReached): return fmt.Sprintf("Quota Reached:\n%v", err), fasthttp.StatusConflict @@ -101,7 +101,7 @@ func formErrorResponse(err error) (string, int) { return fmt.Sprintf("Container Not Found:\n%v", err), fasthttp.StatusNotFound case errors.Is(err, ErrObjectNotFound): return fmt.Sprintf("Object Not Found:\n%v", err), fasthttp.StatusNotFound - case errors.Is(err, layer.ErrNodeNotFound): + case errors.Is(err, tree.ErrNodeNotFound): return fmt.Sprintf("Tree Node Not Found:\n%v", err), fasthttp.StatusNotFound case errors.Is(err, ErrGatewayTimeout): return fmt.Sprintf("Gateway Timeout:\n%v", err), fasthttp.StatusGatewayTimeout diff --git a/internal/layer/tree_service.go b/internal/layer/tree_service.go deleted file mode 100644 index ff80543..0000000 --- a/internal/layer/tree_service.go +++ /dev/null @@ -1,24 +0,0 @@ -package layer - -import ( - "context" - "errors" - - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" - cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" -) - -// TreeService provide interface to interact with tree service using s3 data models. -type TreeService interface { - GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*data.NodeVersion, error) - GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string, latestOnly bool) ([]data.NodeInfo, string, error) - CheckSettingsNodeExists(ctx context.Context, bktInfo *data.BucketInfo) error -} - -var ( - // ErrNodeNotFound is returned from Tree service in case of not found error. - ErrNodeNotFound = errors.New("not found") - - // ErrNodeAccessDenied is returned from Tree service in case of access denied error. - ErrNodeAccessDenied = errors.New("access denied") -) diff --git a/internal/templates/index.gotmpl b/internal/templates/index.gotmpl index b14cc06..4c03404 100644 --- a/internal/templates/index.gotmpl +++ b/internal/templates/index.gotmpl @@ -1,11 +1,9 @@ {{$container := .Container}} -{{ $prefix := trimPrefix .Prefix }}
-