TrueCloudLab/frostfs-http-gw
17
1
Fork 18
Code Issues 10 Pull requests 6 Projects Releases 16 Packages 1 Activity Actions

[#240] Fix s3 index page #240

Open
dkirillov wants to merge 2 commits from dkirillov/frostfs-http-gw:fix-listing into master
pull from: dkirillov/frostfs-http-gw:fix-listing
merge into: TrueCloudLab:master
Conversation 0 Commits 2 Files changed 6 +111 -45
6 changed files with 111 additions and 45 deletions
Download patch file Download diff file Expand all files Collapse all files

42
internal/handler/browse.go
View file

@ -76,13 +76,13 @@ func newListObjectsResponseNative(attrs map[string]string) ResponseObject {
}
}
func getNextDir(filepath, prefix string) string {
func getNextDir(filepath, prefix string) *string {
restPath := strings.Replace(filepath, prefix, "", 1)
index := strings.Index(restPath, "/")
if index == -1 {
return ""
return nil
}
return restPath[:index]
return ptr(restPath[:index])
}
func lastPathElement(path string) string {
@ -143,15 +143,18 @@ func getParent(encPrefix string) string {
if slashIndex == -1 {
return ""
}
return prefix[:slashIndex]
return prefix[:slashIndex+1]
}
func urlencode(path string) string {
var res strings.Builder
prefixParts := strings.Split(path, "/")
for _, prefixPart := range prefixParts {
prefixPart = "/" + url.PathEscape(prefixPart)
for i, prefixPart := range prefixParts {
prefixPart = url.PathEscape(prefixPart)
if i != 0 {
prefixPart = "/" + prefixPart
}
if prefixPart == "/." || prefixPart == "/.." {
prefixPart = url.PathEscape(prefixPart)
}
@ -168,11 +171,16 @@ type GetObjectsResponse struct {
}
func (h *Handler) getDirObjectsS3(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) (*GetObjectsResponse, error) {
if prefix != "" && prefix[len(prefix)-1] == '/' {
prefix = prefix[:len(prefix)-1]
var treePrefix *string
if prefix != "" {
if prefix[len(prefix)-1] == '/' {
treePrefix = ptr(prefix[:len(prefix)-1])
} else {
treePrefix = &prefix
}
}
nodes, err := h.tree.GetSubTreeByPrefix(ctx, bucketInfo, prefix, true)
nodes, err := h.tree.GetSubTreeByPrefix(ctx, bucketInfo, treePrefix, true)
if err != nil {
return nil, err
}
@ -193,14 +201,18 @@ func (h *Handler) getDirObjectsS3(ctx context.Context, bucketInfo *data.BucketIn
if obj.IsDeleteMarker {
continue
}
obj.FilePath = prefix + "/" + obj.FileName
obj.GetURL = "/get/" + bucketInfo.Name + urlencode(obj.FilePath)
obj.FilePath = prefix + obj.FileName
obj.GetURL = "/get/" + bucketInfo.Name + "/" + urlencode(obj.FilePath)
result.objects = append(result.objects, obj)
}
return result, nil
}
func ptr(s string) *string {
return &s
}
func (h *Handler) getDirObjectsNative(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) (*GetObjectsResponse, error) {
basePath := prefix
if basePath != "" && basePath[len(basePath)-1] != '/' {
@ -247,7 +259,7 @@ func (h *Handler) getDirObjectsNative(ctx context.Context, bucketInfo *data.Buck
if _, ok := dirs[objExt.Object.FileName]; ok {
continue
}
objExt.Object.GetURL = "/get/" + bucketInfo.CID.EncodeToString() + urlencode(objExt.Object.FilePath)
objExt.Object.GetURL = "/get/" + bucketInfo.CID.EncodeToString() + "/" + urlencode(objExt.Object.FilePath)
dirs[objExt.Object.FileName] = struct{}{}
} else {
objExt.Object.GetURL = "/get/" + bucketInfo.CID.EncodeToString() + "/" + objExt.Object.OID
@ -319,13 +331,13 @@ func (h *Handler) headDirObject(ctx context.Context, cnrID cid.ID, objID oid.ID,
}
dirname := getNextDir(attrs[object.AttributeFilePath], basePath)
if dirname == "" {
if dirname == nil {
return newListObjectsResponseNative(attrs), nil
}
return ResponseObject{
FileName: dirname,
FilePath: basePath + dirname,
FileName: *dirname,
FilePath: basePath + *dirname,
IsDir: true,
}, nil
}

5
internal/handler/handler.go
View file

@ -325,11 +325,12 @@ func (h *Handler) browseIndexMiddleware(fn ListFunc) MiddlewareFunc {
ctx, span := tracing.StartSpanFromContext(prm.Context, "handler.browseIndex")
defer span.End()
ctx = utils.SetReqLog(ctx, h.reqLogger(ctx).With(
h.reqLogger(ctx).Info(logs.BrowseIndex,
zap.String("bucket", prm.BktInfo.Name),
zap.String("container", prm.BktInfo.CID.EncodeToString()),
zap.String("prefix", prm.Path),
))
logs.TagField(logs.TagDatapath),
)
objects, err := fn(ctx, prm.BktInfo, prm.Path)
if err != nil {

68
internal/handler/handler_test.go
View file

@ -520,15 +520,23 @@ func TestIndex(t *testing.T) {
obj1.SetAttributes(prepareObjectAttributes(object.AttributeFilePath, "prefix/obj1"))
hc.frostfs.objects[cnrID.String()+"/"+obj1ID.String()] = obj1
obj2ID := oidtest.ID()
obj2 := object.New()
obj2.SetID(obj2ID)
obj2.SetPayload([]byte("obj2"))
obj2.SetAttributes(prepareObjectAttributes(object.AttributeFilePath, "/dir/.."))
hc.frostfs.objects[cnrID.String()+"/"+obj2ID.String()] = obj2
hc.tree.containers[cnrID.String()] = containerInfo{
trees: map[string]map[string]nodeResponse{
"system": {"bucket-settings": nodeResponse{nodeID: 1}},
"version": {
"": nodeResponse{}, //root
"<root>": nodeResponse{}, //root
"prefix": nodeResponse{
nodeID: 1,
meta: []nodeMeta{{key: tree.FileNameKey, value: []byte("prefix")}}},
"obj1": nodeResponse{
meta: []nodeMeta{{key: tree.FileNameKey, value: []byte("prefix")}},
},
"prefix/obj1": nodeResponse{
parentID: 1,
nodeID: 2,
meta: []nodeMeta{
@ -536,6 +544,23 @@ func TestIndex(t *testing.T) {
{key: "OID", value: []byte(obj1ID.String())},
},
},
"": nodeResponse{
nodeID: 3,
meta: []nodeMeta{{key: tree.FileNameKey, value: []byte("")}},
},
"/dir": nodeResponse{
parentID: 3,
nodeID: 4,
meta: []nodeMeta{{key: tree.FileNameKey, value: []byte("dir")}},
},
"/dir/..": nodeResponse{
parentID: 4,
nodeID: 5,
meta: []nodeMeta{
{key: tree.FileNameKey, value: []byte("..")},
{key: "OID", value: []byte(obj2ID.String())},
},
},
},
},
}
@ -563,6 +588,21 @@ func TestIndex(t *testing.T) {
r = prepareGetRequest(ctx, "bucket", "dummy")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Contains(t, string(r.Response.Body()), "Index of s3://bucket/dummy")
r = prepareGetRequest(ctx, "bucket", "")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Contains(t, string(r.Response.Body()), `<a href="/get/bucket/">..</a>`)
require.Contains(t, string(r.Response.Body()), `<a href="/get/bucket//">/</a>`)
r = prepareGetRequest(ctx, "bucket", "/")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Contains(t, string(r.Response.Body()), `<a href="/get/bucket/">..</a>`)
require.Contains(t, string(r.Response.Body()), `<a href="/get/bucket//dir/">dir/</a>`)
r = prepareGetRequest(ctx, "bucket", "/dir/")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Contains(t, string(r.Response.Body()), `<a href="/get/bucket//">..</a>`)
require.Contains(t, string(r.Response.Body()), `<a href="/get/bucket//dir%2F..">..</a>`)
})
t.Run("native", func(t *testing.T) {
@ -575,6 +615,13 @@ func TestIndex(t *testing.T) {
obj1.SetAttributes(prepareObjectAttributes(object.AttributeFilePath, "prefix/obj1"))
hc.frostfs.objects[cnrID.String()+"/"+obj1ID.String()] = obj1
obj2ID := oidtest.ID()
obj2 := object.New()
obj2.SetID(obj2ID)
obj2.SetPayload([]byte("obj2"))
obj2.SetAttributes(prepareObjectAttributes(object.AttributeFilePath, "/dir/.."))
hc.frostfs.objects[cnrID.String()+"/"+obj2ID.String()] = obj2
r := prepareGetRequest(ctx, cnrID.EncodeToString(), "prefix/")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode())
@ -598,6 +645,21 @@ func TestIndex(t *testing.T) {
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "dummy")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Contains(t, string(r.Response.Body()), "Index of frostfs://"+cnrID.String()+"/dummy")
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Contains(t, string(r.Response.Body()), `<a href="/get/`+cnrID.String()+`/">..</a>`)
require.Contains(t, string(r.Response.Body()), `<a href="/get/`+cnrID.String()+`//">/</a>`)
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "/")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Contains(t, string(r.Response.Body()), `<a href="/get/`+cnrID.String()+`/">..</a>`)
require.Contains(t, string(r.Response.Body()), `<a href="/get/`+cnrID.String()+`//dir/">dir/</a>`)
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "/dir/")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Contains(t, string(r.Response.Body()), `<a href="/get/`+cnrID.String()+`//">..</a>`)
require.Contains(t, string(r.Response.Body()), `<a href="/get/`+cnrID.String()+`/`+obj2ID.String()+`">..</a>`)
})
}

1
internal/logs/logs.go
View file

@ -127,6 +127,7 @@ const (
EmptyAccessControlRequestMethodHeader = "empty Access-Control-Request-Method request header"
CORSRuleWasNotMatched = "cors rule was not matched"
CouldntCacheCors = "couldn't cache cors"
BrowseIndex = "browse index"
)
// Log messages with the "external_storage" tag.

22
internal/templates/index.gotmpl
View file

@ -56,40 +56,24 @@
</thead>
<tbody>
{{ $parentPrefix := getParent .Prefix }}
{{if $parentPrefix }}
<tr>
<td>
⮐<a href="/get/{{$container}}{{ urlencode $parentPrefix }}/">..</a>
⮐<a href="/get/{{$container}}/{{ urlencode $parentPrefix }}">..</a>
</td>
<td></td>
<td></td>
<td></td>
<td></td>
</tr>
{{else}}
<tr>
<td>
⮐<a href="/get/{{$container}}/">..</a>
</td>
<td></td>
<td></td>
<td></td>
<td></td>
</tr>
{{end}}
{{range .Objects}}
<tr>
<td>
{{if .IsDir}}
🗀
<a href="{{.GetURL}}/">
{{.FileName}}/
</a>
<a href="{{.GetURL}}/">{{.FileName}}/</a>
{{else}}
🗎
<a href="{{ .GetURL }}">
{{.FileName}}
</a>
<a href="{{ .GetURL }}">{{.FileName}}</a>
{{end}}
</td>
<td>{{.OID}}</td>

18
tree/tree.go
View file

@ -323,17 +323,23 @@ func pathFromName(objectName string) []string {
return strings.Split(objectName, separator)
}
func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string, latestOnly bool) ([]data.NodeInfo, error) {
func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix *string, latestOnly bool) ([]data.NodeInfo, error) {
ctx, span := tracing.StartSpanFromContext(ctx, "tree.GetSubTreeByPrefix")
defer span.End()
rootID, err := c.getPrefixNodeID(ctx, bktInfo, versionTree, strings.Split(prefix, separator))
if err != nil {
if errors.Is(err, ErrNodeNotFound) {
return nil, nil
rootID := []uint64{0}
var err error
if prefix != nil {
rootID, err = c.getPrefixNodeID(ctx, bktInfo, versionTree, strings.Split(*prefix, separator))
if err != nil {
if errors.Is(err, ErrNodeNotFound) {
return nil, nil
}
return nil, err
}
return nil, err
}
subTree, err := c.service.GetSubTree(ctx, bktInfo, versionTree, rootID, 2, false)
if err != nil {
if errors.Is(err, ErrNodeNotFound) {