[#240] Fix s3 index page #240
6 changed files with 111 additions and 45 deletions
|
@ -76,13 +76,13 @@ func newListObjectsResponseNative(attrs map[string]string) ResponseObject {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func getNextDir(filepath, prefix string) string {
|
func getNextDir(filepath, prefix string) *string {
|
||||||
restPath := strings.Replace(filepath, prefix, "", 1)
|
restPath := strings.Replace(filepath, prefix, "", 1)
|
||||||
index := strings.Index(restPath, "/")
|
index := strings.Index(restPath, "/")
|
||||||
if index == -1 {
|
if index == -1 {
|
||||||
return ""
|
return nil
|
||||||
}
|
}
|
||||||
return restPath[:index]
|
return ptr(restPath[:index])
|
||||||
}
|
}
|
||||||
|
|
||||||
func lastPathElement(path string) string {
|
func lastPathElement(path string) string {
|
||||||
|
@ -143,15 +143,18 @@ func getParent(encPrefix string) string {
|
||||||
if slashIndex == -1 {
|
if slashIndex == -1 {
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
return prefix[:slashIndex]
|
return prefix[:slashIndex+1]
|
||||||
}
|
}
|
||||||
|
|
||||||
func urlencode(path string) string {
|
func urlencode(path string) string {
|
||||||
var res strings.Builder
|
var res strings.Builder
|
||||||
|
|
||||||
prefixParts := strings.Split(path, "/")
|
prefixParts := strings.Split(path, "/")
|
||||||
for _, prefixPart := range prefixParts {
|
for i, prefixPart := range prefixParts {
|
||||||
prefixPart = "/" + url.PathEscape(prefixPart)
|
prefixPart = url.PathEscape(prefixPart)
|
||||||
|
if i != 0 {
|
||||||
|
prefixPart = "/" + prefixPart
|
||||||
|
}
|
||||||
if prefixPart == "/." || prefixPart == "/.." {
|
if prefixPart == "/." || prefixPart == "/.." {
|
||||||
prefixPart = url.PathEscape(prefixPart)
|
prefixPart = url.PathEscape(prefixPart)
|
||||||
}
|
}
|
||||||
|
@ -168,11 +171,16 @@ type GetObjectsResponse struct {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *Handler) getDirObjectsS3(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) (*GetObjectsResponse, error) {
|
func (h *Handler) getDirObjectsS3(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) (*GetObjectsResponse, error) {
|
||||||
if prefix != "" && prefix[len(prefix)-1] == '/' {
|
var treePrefix *string
|
||||||
prefix = prefix[:len(prefix)-1]
|
if prefix != "" {
|
||||||
|
if prefix[len(prefix)-1] == '/' {
|
||||||
|
treePrefix = ptr(prefix[:len(prefix)-1])
|
||||||
|
} else {
|
||||||
|
treePrefix = &prefix
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
nodes, err := h.tree.GetSubTreeByPrefix(ctx, bucketInfo, prefix, true)
|
nodes, err := h.tree.GetSubTreeByPrefix(ctx, bucketInfo, treePrefix, true)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
@ -193,14 +201,18 @@ func (h *Handler) getDirObjectsS3(ctx context.Context, bucketInfo *data.BucketIn
|
||||||
if obj.IsDeleteMarker {
|
if obj.IsDeleteMarker {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
obj.FilePath = prefix + "/" + obj.FileName
|
obj.FilePath = prefix + obj.FileName
|
||||||
obj.GetURL = "/get/" + bucketInfo.Name + urlencode(obj.FilePath)
|
obj.GetURL = "/get/" + bucketInfo.Name + "/" + urlencode(obj.FilePath)
|
||||||
result.objects = append(result.objects, obj)
|
result.objects = append(result.objects, obj)
|
||||||
}
|
}
|
||||||
|
|
||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func ptr(s string) *string {
|
||||||
|
return &s
|
||||||
|
}
|
||||||
|
|
||||||
func (h *Handler) getDirObjectsNative(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) (*GetObjectsResponse, error) {
|
func (h *Handler) getDirObjectsNative(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) (*GetObjectsResponse, error) {
|
||||||
basePath := prefix
|
basePath := prefix
|
||||||
if basePath != "" && basePath[len(basePath)-1] != '/' {
|
if basePath != "" && basePath[len(basePath)-1] != '/' {
|
||||||
|
@ -247,7 +259,7 @@ func (h *Handler) getDirObjectsNative(ctx context.Context, bucketInfo *data.Buck
|
||||||
if _, ok := dirs[objExt.Object.FileName]; ok {
|
if _, ok := dirs[objExt.Object.FileName]; ok {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
objExt.Object.GetURL = "/get/" + bucketInfo.CID.EncodeToString() + urlencode(objExt.Object.FilePath)
|
objExt.Object.GetURL = "/get/" + bucketInfo.CID.EncodeToString() + "/" + urlencode(objExt.Object.FilePath)
|
||||||
dirs[objExt.Object.FileName] = struct{}{}
|
dirs[objExt.Object.FileName] = struct{}{}
|
||||||
} else {
|
} else {
|
||||||
objExt.Object.GetURL = "/get/" + bucketInfo.CID.EncodeToString() + "/" + objExt.Object.OID
|
objExt.Object.GetURL = "/get/" + bucketInfo.CID.EncodeToString() + "/" + objExt.Object.OID
|
||||||
|
@ -319,13 +331,13 @@ func (h *Handler) headDirObject(ctx context.Context, cnrID cid.ID, objID oid.ID,
|
||||||
}
|
}
|
||||||
|
|
||||||
dirname := getNextDir(attrs[object.AttributeFilePath], basePath)
|
dirname := getNextDir(attrs[object.AttributeFilePath], basePath)
|
||||||
if dirname == "" {
|
if dirname == nil {
|
||||||
return newListObjectsResponseNative(attrs), nil
|
return newListObjectsResponseNative(attrs), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
return ResponseObject{
|
return ResponseObject{
|
||||||
FileName: dirname,
|
FileName: *dirname,
|
||||||
FilePath: basePath + dirname,
|
FilePath: basePath + *dirname,
|
||||||
IsDir: true,
|
IsDir: true,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -325,11 +325,12 @@ func (h *Handler) browseIndexMiddleware(fn ListFunc) MiddlewareFunc {
|
||||||
ctx, span := tracing.StartSpanFromContext(prm.Context, "handler.browseIndex")
|
ctx, span := tracing.StartSpanFromContext(prm.Context, "handler.browseIndex")
|
||||||
defer span.End()
|
defer span.End()
|
||||||
|
|
||||||
ctx = utils.SetReqLog(ctx, h.reqLogger(ctx).With(
|
h.reqLogger(ctx).Info(logs.BrowseIndex,
|
||||||
zap.String("bucket", prm.BktInfo.Name),
|
zap.String("bucket", prm.BktInfo.Name),
|
||||||
zap.String("container", prm.BktInfo.CID.EncodeToString()),
|
zap.String("container", prm.BktInfo.CID.EncodeToString()),
|
||||||
zap.String("prefix", prm.Path),
|
zap.String("prefix", prm.Path),
|
||||||
))
|
logs.TagField(logs.TagDatapath),
|
||||||
|
)
|
||||||
|
|
||||||
objects, err := fn(ctx, prm.BktInfo, prm.Path)
|
objects, err := fn(ctx, prm.BktInfo, prm.Path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
|
@ -520,15 +520,23 @@ func TestIndex(t *testing.T) {
|
||||||
obj1.SetAttributes(prepareObjectAttributes(object.AttributeFilePath, "prefix/obj1"))
|
obj1.SetAttributes(prepareObjectAttributes(object.AttributeFilePath, "prefix/obj1"))
|
||||||
hc.frostfs.objects[cnrID.String()+"/"+obj1ID.String()] = obj1
|
hc.frostfs.objects[cnrID.String()+"/"+obj1ID.String()] = obj1
|
||||||
|
|
||||||
|
obj2ID := oidtest.ID()
|
||||||
|
obj2 := object.New()
|
||||||
|
obj2.SetID(obj2ID)
|
||||||
|
obj2.SetPayload([]byte("obj2"))
|
||||||
|
obj2.SetAttributes(prepareObjectAttributes(object.AttributeFilePath, "/dir/.."))
|
||||||
|
hc.frostfs.objects[cnrID.String()+"/"+obj2ID.String()] = obj2
|
||||||
|
|
||||||
hc.tree.containers[cnrID.String()] = containerInfo{
|
hc.tree.containers[cnrID.String()] = containerInfo{
|
||||||
trees: map[string]map[string]nodeResponse{
|
trees: map[string]map[string]nodeResponse{
|
||||||
"system": {"bucket-settings": nodeResponse{nodeID: 1}},
|
"system": {"bucket-settings": nodeResponse{nodeID: 1}},
|
||||||
"version": {
|
"version": {
|
||||||
"": nodeResponse{}, //root
|
"<root>": nodeResponse{}, //root
|
||||||
"prefix": nodeResponse{
|
"prefix": nodeResponse{
|
||||||
nodeID: 1,
|
nodeID: 1,
|
||||||
meta: []nodeMeta{{key: tree.FileNameKey, value: []byte("prefix")}}},
|
meta: []nodeMeta{{key: tree.FileNameKey, value: []byte("prefix")}},
|
||||||
"obj1": nodeResponse{
|
},
|
||||||
|
"prefix/obj1": nodeResponse{
|
||||||
parentID: 1,
|
parentID: 1,
|
||||||
nodeID: 2,
|
nodeID: 2,
|
||||||
meta: []nodeMeta{
|
meta: []nodeMeta{
|
||||||
|
@ -536,6 +544,23 @@ func TestIndex(t *testing.T) {
|
||||||
{key: "OID", value: []byte(obj1ID.String())},
|
{key: "OID", value: []byte(obj1ID.String())},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
"": nodeResponse{
|
||||||
|
nodeID: 3,
|
||||||
|
meta: []nodeMeta{{key: tree.FileNameKey, value: []byte("")}},
|
||||||
|
},
|
||||||
|
"/dir": nodeResponse{
|
||||||
|
parentID: 3,
|
||||||
|
nodeID: 4,
|
||||||
|
meta: []nodeMeta{{key: tree.FileNameKey, value: []byte("dir")}},
|
||||||
|
},
|
||||||
|
"/dir/..": nodeResponse{
|
||||||
|
parentID: 4,
|
||||||
|
nodeID: 5,
|
||||||
|
meta: []nodeMeta{
|
||||||
|
{key: tree.FileNameKey, value: []byte("..")},
|
||||||
|
{key: "OID", value: []byte(obj2ID.String())},
|
||||||
|
},
|
||||||
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
@ -563,6 +588,21 @@ func TestIndex(t *testing.T) {
|
||||||
r = prepareGetRequest(ctx, "bucket", "dummy")
|
r = prepareGetRequest(ctx, "bucket", "dummy")
|
||||||
hc.Handler().DownloadByAddressOrBucketName(r)
|
hc.Handler().DownloadByAddressOrBucketName(r)
|
||||||
require.Contains(t, string(r.Response.Body()), "Index of s3://bucket/dummy")
|
require.Contains(t, string(r.Response.Body()), "Index of s3://bucket/dummy")
|
||||||
|
|
||||||
|
r = prepareGetRequest(ctx, "bucket", "")
|
||||||
|
hc.Handler().DownloadByAddressOrBucketName(r)
|
||||||
|
require.Contains(t, string(r.Response.Body()), `<a href="/get/bucket/">..</a>`)
|
||||||
|
require.Contains(t, string(r.Response.Body()), `<a href="/get/bucket//">/</a>`)
|
||||||
|
|
||||||
|
r = prepareGetRequest(ctx, "bucket", "/")
|
||||||
|
hc.Handler().DownloadByAddressOrBucketName(r)
|
||||||
|
require.Contains(t, string(r.Response.Body()), `<a href="/get/bucket/">..</a>`)
|
||||||
|
require.Contains(t, string(r.Response.Body()), `<a href="/get/bucket//dir/">dir/</a>`)
|
||||||
|
|
||||||
|
r = prepareGetRequest(ctx, "bucket", "/dir/")
|
||||||
|
hc.Handler().DownloadByAddressOrBucketName(r)
|
||||||
|
require.Contains(t, string(r.Response.Body()), `<a href="/get/bucket//">..</a>`)
|
||||||
|
require.Contains(t, string(r.Response.Body()), `<a href="/get/bucket//dir%2F..">..</a>`)
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("native", func(t *testing.T) {
|
t.Run("native", func(t *testing.T) {
|
||||||
|
@ -575,6 +615,13 @@ func TestIndex(t *testing.T) {
|
||||||
obj1.SetAttributes(prepareObjectAttributes(object.AttributeFilePath, "prefix/obj1"))
|
obj1.SetAttributes(prepareObjectAttributes(object.AttributeFilePath, "prefix/obj1"))
|
||||||
hc.frostfs.objects[cnrID.String()+"/"+obj1ID.String()] = obj1
|
hc.frostfs.objects[cnrID.String()+"/"+obj1ID.String()] = obj1
|
||||||
|
|
||||||
|
obj2ID := oidtest.ID()
|
||||||
|
obj2 := object.New()
|
||||||
|
obj2.SetID(obj2ID)
|
||||||
|
obj2.SetPayload([]byte("obj2"))
|
||||||
|
obj2.SetAttributes(prepareObjectAttributes(object.AttributeFilePath, "/dir/.."))
|
||||||
|
hc.frostfs.objects[cnrID.String()+"/"+obj2ID.String()] = obj2
|
||||||
|
|
||||||
r := prepareGetRequest(ctx, cnrID.EncodeToString(), "prefix/")
|
r := prepareGetRequest(ctx, cnrID.EncodeToString(), "prefix/")
|
||||||
hc.Handler().DownloadByAddressOrBucketName(r)
|
hc.Handler().DownloadByAddressOrBucketName(r)
|
||||||
require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode())
|
require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode())
|
||||||
|
@ -598,6 +645,21 @@ func TestIndex(t *testing.T) {
|
||||||
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "dummy")
|
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "dummy")
|
||||||
hc.Handler().DownloadByAddressOrBucketName(r)
|
hc.Handler().DownloadByAddressOrBucketName(r)
|
||||||
require.Contains(t, string(r.Response.Body()), "Index of frostfs://"+cnrID.String()+"/dummy")
|
require.Contains(t, string(r.Response.Body()), "Index of frostfs://"+cnrID.String()+"/dummy")
|
||||||
|
|
||||||
|
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "")
|
||||||
|
hc.Handler().DownloadByAddressOrBucketName(r)
|
||||||
|
require.Contains(t, string(r.Response.Body()), `<a href="/get/`+cnrID.String()+`/">..</a>`)
|
||||||
|
require.Contains(t, string(r.Response.Body()), `<a href="/get/`+cnrID.String()+`//">/</a>`)
|
||||||
|
|
||||||
|
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "/")
|
||||||
|
hc.Handler().DownloadByAddressOrBucketName(r)
|
||||||
|
require.Contains(t, string(r.Response.Body()), `<a href="/get/`+cnrID.String()+`/">..</a>`)
|
||||||
|
require.Contains(t, string(r.Response.Body()), `<a href="/get/`+cnrID.String()+`//dir/">dir/</a>`)
|
||||||
|
|
||||||
|
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "/dir/")
|
||||||
|
hc.Handler().DownloadByAddressOrBucketName(r)
|
||||||
|
require.Contains(t, string(r.Response.Body()), `<a href="/get/`+cnrID.String()+`//">..</a>`)
|
||||||
|
require.Contains(t, string(r.Response.Body()), `<a href="/get/`+cnrID.String()+`/`+obj2ID.String()+`">..</a>`)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -127,6 +127,7 @@ const (
|
||||||
EmptyAccessControlRequestMethodHeader = "empty Access-Control-Request-Method request header"
|
EmptyAccessControlRequestMethodHeader = "empty Access-Control-Request-Method request header"
|
||||||
CORSRuleWasNotMatched = "cors rule was not matched"
|
CORSRuleWasNotMatched = "cors rule was not matched"
|
||||||
CouldntCacheCors = "couldn't cache cors"
|
CouldntCacheCors = "couldn't cache cors"
|
||||||
|
BrowseIndex = "browse index"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Log messages with the "external_storage" tag.
|
// Log messages with the "external_storage" tag.
|
||||||
|
|
|
@ -56,40 +56,24 @@
|
||||||
</thead>
|
</thead>
|
||||||
<tbody>
|
<tbody>
|
||||||
{{ $parentPrefix := getParent .Prefix }}
|
{{ $parentPrefix := getParent .Prefix }}
|
||||||
{{if $parentPrefix }}
|
|
||||||
<tr>
|
<tr>
|
||||||
<td>
|
<td>
|
||||||
⮐<a href="/get/{{$container}}{{ urlencode $parentPrefix }}/">..</a>
|
⮐<a href="/get/{{$container}}/{{ urlencode $parentPrefix }}">..</a>
|
||||||
</td>
|
</td>
|
||||||
<td></td>
|
<td></td>
|
||||||
<td></td>
|
<td></td>
|
||||||
<td></td>
|
<td></td>
|
||||||
<td></td>
|
<td></td>
|
||||||
</tr>
|
</tr>
|
||||||
{{else}}
|
|
||||||
<tr>
|
|
||||||
<td>
|
|
||||||
⮐<a href="/get/{{$container}}/">..</a>
|
|
||||||
</td>
|
|
||||||
<td></td>
|
|
||||||
<td></td>
|
|
||||||
<td></td>
|
|
||||||
<td></td>
|
|
||||||
</tr>
|
|
||||||
{{end}}
|
|
||||||
{{range .Objects}}
|
{{range .Objects}}
|
||||||
<tr>
|
<tr>
|
||||||
<td>
|
<td>
|
||||||
{{if .IsDir}}
|
{{if .IsDir}}
|
||||||
🗀
|
🗀
|
||||||
<a href="{{.GetURL}}/">
|
<a href="{{.GetURL}}/">{{.FileName}}/</a>
|
||||||
{{.FileName}}/
|
|
||||||
</a>
|
|
||||||
{{else}}
|
{{else}}
|
||||||
🗎
|
🗎
|
||||||
<a href="{{ .GetURL }}">
|
<a href="{{ .GetURL }}">{{.FileName}}</a>
|
||||||
{{.FileName}}
|
|
||||||
</a>
|
|
||||||
{{end}}
|
{{end}}
|
||||||
</td>
|
</td>
|
||||||
<td>{{.OID}}</td>
|
<td>{{.OID}}</td>
|
||||||
|
|
10
tree/tree.go
10
tree/tree.go
|
@ -323,17 +323,23 @@ func pathFromName(objectName string) []string {
|
||||||
return strings.Split(objectName, separator)
|
return strings.Split(objectName, separator)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string, latestOnly bool) ([]data.NodeInfo, error) {
|
func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix *string, latestOnly bool) ([]data.NodeInfo, error) {
|
||||||
ctx, span := tracing.StartSpanFromContext(ctx, "tree.GetSubTreeByPrefix")
|
ctx, span := tracing.StartSpanFromContext(ctx, "tree.GetSubTreeByPrefix")
|
||||||
defer span.End()
|
defer span.End()
|
||||||
|
|
||||||
rootID, err := c.getPrefixNodeID(ctx, bktInfo, versionTree, strings.Split(prefix, separator))
|
rootID := []uint64{0}
|
||||||
|
var err error
|
||||||
|
|
||||||
|
if prefix != nil {
|
||||||
|
rootID, err = c.getPrefixNodeID(ctx, bktInfo, versionTree, strings.Split(*prefix, separator))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if errors.Is(err, ErrNodeNotFound) {
|
if errors.Is(err, ErrNodeNotFound) {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
subTree, err := c.service.GetSubTree(ctx, bktInfo, versionTree, rootID, 2, false)
|
subTree, err := c.service.GetSubTree(ctx, bktInfo, versionTree, rootID, 2, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if errors.Is(err, ErrNodeNotFound) {
|
if errors.Is(err, ErrNodeNotFound) {
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue