From 65be38831c5f04aa083e9874324c830f8b02b2f4 Mon Sep 17 00:00:00 2001
From: Denis Kirillov <denis@nspcc.ru>
Date: Tue, 29 Jun 2021 12:59:33 +0300
Subject: [PATCH] [#98] Supported listObjects delimiter
Signed-off-by: Denis Kirillov <denis@nspcc.ru>
---
api/handler/list.go | 10 +--
api/layer/layer.go | 30 ++++++---
api/layer/util.go | 77 +++++++++++-----------
api/layer/util_test.go | 141 +++++++++++++++--------------------------
4 files changed, 113 insertions(+), 145 deletions(-)
diff --git a/api/handler/list.go b/api/handler/list.go
index 35e74a79..a05257fb 100644
--- a/api/handler/list.go
+++ b/api/handler/list.go
@@ -14,7 +14,7 @@ import (
type listObjectsArgs struct {
Bucket string
- Delimeter string
+ Delimiter string
Encode string
Marker string
StartAfter string
@@ -124,7 +124,7 @@ func (h *handler) listObjects(w http.ResponseWriter, r *http.Request) (*listObje
Bucket: arg.Bucket,
Prefix: arg.Prefix,
MaxKeys: arg.MaxKeys,
- Delimiter: arg.Delimeter,
+ Delimiter: arg.Delimiter,
Marker: marker,
})
if err != nil {
@@ -170,7 +170,7 @@ func encodeV1(arg *listObjectsArgs, list *layer.ListObjectsInfo) *ListObjectsRes
Marker: arg.Marker,
Prefix: arg.Prefix,
MaxKeys: arg.MaxKeys,
- Delimiter: arg.Delimeter,
+ Delimiter: arg.Delimiter,
IsTruncated: list.IsTruncated,
NextMarker: list.NextMarker,
@@ -230,7 +230,7 @@ func encodeV2(arg *listObjectsArgs, list *layer.ListObjectsInfo) *ListObjectsV2R
Prefix: arg.Prefix,
KeyCount: len(list.Objects),
MaxKeys: arg.MaxKeys,
- Delimiter: arg.Delimeter,
+ Delimiter: arg.Delimiter,
StartAfter: arg.StartAfter,
IsTruncated: list.IsTruncated,
@@ -281,7 +281,7 @@ func parseListObjectArgs(r *http.Request) (*listObjectsArgs, error) {
res.Prefix = r.URL.Query().Get("prefix")
res.Marker = r.URL.Query().Get("marker")
- res.Delimeter = r.URL.Query().Get("delimiter")
+ res.Delimiter = r.URL.Query().Get("delimiter")
res.Encode = r.URL.Query().Get("encoding-type")
res.StartAfter = r.URL.Query().Get("start-after")
apiVersionStr := r.URL.Query().Get("list-type")
diff --git a/api/layer/layer.go b/api/layer/layer.go
index a81ba923..30593e50 100644
--- a/api/layer/layer.go
+++ b/api/layer/layer.go
@@ -201,7 +201,7 @@ func (n *layer) ListObjects(ctx context.Context, p *ListObjectsParams) (*ListObj
bkt *BucketInfo
ids []*object.ID
result ListObjectsInfo
- uniqNames = make(map[string]struct{})
+ uniqNames = make(map[string]bool)
)
if bkt, err = n.GetBucketInfo(ctx, p.Bucket); err != nil {
@@ -250,16 +250,18 @@ func (n *layer) ListObjects(ctx context.Context, p *ListObjectsParams) (*ListObj
// and look for entities after prefix. If entity does not have any
// sub-entities, then it is a file, else directory.
- if oi := objectInfoFromMeta(bkt, meta, p.Prefix); oi != nil {
+ if oi := objectInfoFromMeta(bkt, meta, p.Prefix, p.Delimiter); oi != nil {
// use only unique dir names
- if _, ok := uniqNames[oi.Name]; !ok {
- if len(p.Marker) > 0 && oi.Name <= p.Marker {
- continue
- }
- uniqNames[oi.Name] = struct{}{}
-
- result.Objects = append(result.Objects, oi)
+ if _, ok := uniqNames[oi.Name]; ok {
+ continue
}
+ if len(p.Marker) > 0 && oi.Name <= p.Marker {
+ continue
+ }
+
+ uniqNames[oi.Name] = oi.isDir
+
+ result.Objects = append(result.Objects, oi)
}
}
@@ -272,6 +274,14 @@ func (n *layer) ListObjects(ctx context.Context, p *ListObjectsParams) (*ListObj
result.Objects = result.Objects[:p.MaxKeys]
result.NextMarker = result.Objects[len(result.Objects)-1].Name
}
+
+ for i, oi := range result.Objects {
+ if isDir := uniqNames[oi.Name]; isDir {
+ result.Objects = append(result.Objects[:i], result.Objects[i+1:]...)
+ result.Prefixes = append(result.Prefixes, oi.Name)
+ }
+ }
+
return &result, nil
}
@@ -358,7 +368,7 @@ func (n *layer) GetObjectInfo(ctx context.Context, bucketName, filename string)
return nil, err
}
- return objectInfoFromMeta(bkt, meta, rootSeparator), nil
+ return objectInfoFromMeta(bkt, meta, "", ""), nil
}
// PutObject into storage.
diff --git a/api/layer/util.go b/api/layer/util.go
index ba60cc7c..90f9ddb4 100644
--- a/api/layer/util.go
+++ b/api/layer/util.go
@@ -55,11 +55,8 @@ type (
}
)
-const (
- rootSeparator = "root://"
- // PathSeparator is a path components separator string.
- PathSeparator = string(os.PathSeparator)
-)
+// PathSeparator is a path components separator string.
+const PathSeparator = string(os.PathSeparator)
func userHeaders(attrs []*object.Attribute) map[string]string {
result := make(map[string]string, len(attrs))
@@ -71,30 +68,25 @@ func userHeaders(attrs []*object.Attribute) map[string]string {
return result
}
-func objectInfoFromMeta(bkt *BucketInfo, meta *object.Object, prefix string) *ObjectInfo {
+func objectInfoFromMeta(bkt *BucketInfo, meta *object.Object, prefix, delimiter string) *ObjectInfo {
var (
- isDir bool
- size int64
- mimeType string
- creation time.Time
- filename = meta.ID().String()
- name, dirname = nameFromObject(meta)
+ isDir bool
+ size int64
+ mimeType string
+ creation time.Time
+ filename = filenameFromObject(meta)
)
- if !strings.HasPrefix(dirname, prefix) && prefix != rootSeparator {
+ if !strings.HasPrefix(filename, prefix) {
return nil
}
- if ln := len(prefix); ln > 0 && prefix[ln-1:] != PathSeparator {
- prefix += PathSeparator
- }
-
userHeaders := userHeaders(meta.Attributes())
- if val, ok := userHeaders[object.AttributeFileName]; ok {
- filename = val
- delete(userHeaders, object.AttributeFileName)
+ delete(userHeaders, object.AttributeFileName)
+ if contentType, ok := userHeaders[object.AttributeContentType]; ok {
+ mimeType = contentType
+ delete(userHeaders, object.AttributeContentType)
}
-
if val, ok := userHeaders[object.AttributeTimestamp]; !ok {
// ignore empty value
} else if dt, err := strconv.ParseInt(val, 10, 64); err == nil {
@@ -102,20 +94,18 @@ func objectInfoFromMeta(bkt *BucketInfo, meta *object.Object, prefix string) *Ob
delete(userHeaders, object.AttributeTimestamp)
}
- tail := strings.TrimPrefix(dirname, prefix)
- index := strings.Index(tail, PathSeparator)
-
- if prefix == rootSeparator {
- size = int64(meta.PayloadSize())
- mimeType = http.DetectContentType(meta.Payload())
- } else if index < 0 {
- filename = name
- size = int64(meta.PayloadSize())
- mimeType = http.DetectContentType(meta.Payload())
+ if len(delimiter) > 0 {
+ tail := strings.TrimPrefix(filename, prefix)
+ index := strings.Index(tail, delimiter)
+ if index >= 0 {
+ isDir = true
+ filename = prefix + tail[:index+1]
+ userHeaders = nil
+ } else {
+ size, mimeType = getSizeAndMimeType(meta, mimeType)
+ }
} else {
- isDir = true
- filename = tail[:index] + PathSeparator
- userHeaders = nil
+ size, mimeType = getSizeAndMimeType(meta, mimeType)
}
return &ObjectInfo{
@@ -132,18 +122,23 @@ func objectInfoFromMeta(bkt *BucketInfo, meta *object.Object, prefix string) *Ob
}
}
-func nameFromObject(o *object.Object) (string, string) {
- var name = o.ID().String()
+func getSizeAndMimeType(meta *object.Object, contentType string) (size int64, mimeType string) {
+ size = int64(meta.PayloadSize())
+ mimeType = contentType
+ if len(mimeType) == 0 {
+ mimeType = http.DetectContentType(meta.Payload())
+ }
+ return
+}
+func filenameFromObject(o *object.Object) string {
+ var name = o.ID().String()
for _, attr := range o.Attributes() {
if attr.Key() == object.AttributeFileName {
- name = attr.Value()
-
- break
+ return attr.Value()
}
}
-
- return NameFromString(name)
+ return name
}
// NameFromString splits name into base file name and directory path.
diff --git a/api/layer/util_test.go b/api/layer/util_test.go
index 151b8f7c..71ad13b2 100644
--- a/api/layer/util_test.go
+++ b/api/layer/util_test.go
@@ -3,7 +3,6 @@ package layer
import (
"net/http"
"strconv"
- "strings"
"testing"
"time"
@@ -40,7 +39,7 @@ func newTestObject(oid *object.ID, bkt *BucketInfo, name string) *object.Object
return raw.Object()
}
-func newTestInfo(oid *object.ID, bkt *BucketInfo, name, prefix string) *ObjectInfo {
+func newTestInfo(oid *object.ID, bkt *BucketInfo, name string, isDir bool) *ObjectInfo {
info := &ObjectInfo{
id: oid,
Name: name,
@@ -52,34 +51,16 @@ func newTestInfo(oid *object.ID, bkt *BucketInfo, name, prefix string) *ObjectIn
Headers: make(map[string]string),
}
- if prefix == rootSeparator {
- return info
- }
-
- _, dirname := testNameFromObjectName(name)
- if ln := len(prefix); ln > 0 && prefix[ln-1:] != PathSeparator {
- prefix += PathSeparator
- }
-
- tail := strings.TrimPrefix(dirname, prefix)
- if index := strings.Index(tail, PathSeparator); index >= 0 {
+ if isDir {
info.isDir = true
-
info.Size = 0
info.ContentType = ""
- info.Name = tail[:index+1]
info.Headers = nil
}
return info
}
-func testNameFromObjectName(name string) (string, string) {
- ind := strings.LastIndex(name, PathSeparator)
-
- return name[ind+1:], name[:ind+1]
-}
-
func Test_objectInfoFromMeta(t *testing.T) {
uid := owner.NewID()
oid := object.NewID()
@@ -93,99 +74,81 @@ func Test_objectInfoFromMeta(t *testing.T) {
}
cases := []struct {
- name string
- prefix string
- result *ObjectInfo
- object *object.Object
-
- infoName string
+ name string
+ prefix string
+ result *ObjectInfo
+ object *object.Object
+ delimiter string
}{
{
- name: "test.jpg",
- prefix: "",
- infoName: "test.jpg",
- result: newTestInfo(oid, bkt, "test.jpg", ""),
- object: newTestObject(oid, bkt, "test.jpg"),
+ name: "small.jpg",
+ result: newTestInfo(oid, bkt, "small.jpg", false),
+ object: newTestObject(oid, bkt, "small.jpg"),
},
-
{
- name: "test/small.jpg",
- prefix: "",
- infoName: "test/",
- result: newTestInfo(oid, bkt, "test/small.jpg", ""),
- object: newTestObject(oid, bkt, "test/small.jpg"),
+ name: "small.jpg not matched prefix",
+ prefix: "big",
+ result: nil,
+ object: newTestObject(oid, bkt, "small.jpg"),
},
-
{
- name: "test/small.jpg raw",
- prefix: rootSeparator,
- infoName: "test/small.jpg",
- result: newTestInfo(oid, bkt, "test/small.jpg", rootSeparator),
- object: newTestObject(oid, bkt, "test/small.jpg"),
+ name: "small.jpg delimiter",
+ delimiter: "/",
+ result: newTestInfo(oid, bkt, "small.jpg", false),
+ object: newTestObject(oid, bkt, "small.jpg"),
},
-
{
- name: "test/a/b/c/d/e/f/g/h/small.jpg",
- prefix: "",
- infoName: "test/",
- result: newTestInfo(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg", ""),
- object: newTestObject(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg"),
+ name: "test/small.jpg",
+ result: newTestInfo(oid, bkt, "test/small.jpg", false),
+ object: newTestObject(oid, bkt, "test/small.jpg"),
},
-
{
- name: "test/a/b/c/d/e/f/g/h/small.jpg",
- prefix: "test",
- infoName: "a/",
- result: newTestInfo(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg", "test"),
- object: newTestObject(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg"),
+ name: "test/small.jpg with prefix and delimiter",
+ prefix: "test/",
+ delimiter: "/",
+ result: newTestInfo(oid, bkt, "test/small.jpg", false),
+ object: newTestObject(oid, bkt, "test/small.jpg"),
},
-
{
- name: "test/a/b/c/d/e/f/g/h/small.jpg",
- prefix: "test/a",
- infoName: "b/",
- result: newTestInfo(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg", "test/a"),
- object: newTestObject(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg"),
+ name: "a/b/small.jpg",
+ prefix: "a",
+ result: newTestInfo(oid, bkt, "a/b/small.jpg", false),
+ object: newTestObject(oid, bkt, "a/b/small.jpg"),
},
-
{
- name: "test/a/b/c/d/e/f/g/h/small.jpg",
- prefix: "test/a/b",
- infoName: "c/",
- result: newTestInfo(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg", "test/a/b"),
- object: newTestObject(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg"),
+ name: "a/b/small.jpg",
+ prefix: "a/",
+ delimiter: "/",
+ result: newTestInfo(oid, bkt, "a/b/", true),
+ object: newTestObject(oid, bkt, "a/b/small.jpg"),
},
-
{
- name: "test/a/b/c/d/e/f/g/h/small.jpg with slash",
- prefix: "test/a/b/",
- infoName: "c/",
- result: newTestInfo(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg", "test/a/b/"),
- object: newTestObject(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg"),
+ name: "a/b/c/small.jpg",
+ prefix: "a/",
+ delimiter: "/",
+ result: newTestInfo(oid, bkt, "a/b/", true),
+ object: newTestObject(oid, bkt, "a/b/c/small.jpg"),
},
-
{
- name: "test/a/b/c/d/e/f/g/h/small.jpg",
- prefix: "test/a/b/c",
- infoName: "d/",
- result: newTestInfo(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg", "test/a/b/c"),
- object: newTestObject(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg"),
+ name: "a/b/c/small.jpg",
+ prefix: "a/b/c/s",
+ delimiter: "/",
+ result: newTestInfo(oid, bkt, "a/b/c/small.jpg", false),
+ object: newTestObject(oid, bkt, "a/b/c/small.jpg"),
},
-
{
- name: "test/a/b/c/d/e/f/g/h/small.jpg",
- prefix: "test/a/b/c/d",
- infoName: "e/",
- result: newTestInfo(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg", "test/a/b/c/d"),
- object: newTestObject(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg"),
+ name: "a/b/c/big.jpg",
+ prefix: "a/b/",
+ delimiter: "/",
+ result: newTestInfo(oid, bkt, "a/b/c/", true),
+ object: newTestObject(oid, bkt, "a/b/c/big.jpg"),
},
}
for _, tc := range cases {
- t.Run(tc.name+"_"+tc.infoName, func(t *testing.T) {
- info := objectInfoFromMeta(bkt, tc.object, tc.prefix)
+ t.Run(tc.name, func(t *testing.T) {
+ info := objectInfoFromMeta(bkt, tc.object, tc.prefix, tc.delimiter)
require.Equal(t, tc.result, info)
- require.Equal(t, tc.infoName, info.Name)
})
}
}