Merge pull request #115 from KirillovDenis/feature/98-support_delimiter

[#98] Supported listObjects delimiter
This commit is contained in:
Angira Kekteeva 2021-06-30 12:43:10 +03:00 committed by GitHub
commit d2bed40c19
4 changed files with 113 additions and 145 deletions

View file

@ -14,7 +14,7 @@ import (
type listObjectsArgs struct { type listObjectsArgs struct {
Bucket string Bucket string
Delimeter string Delimiter string
Encode string Encode string
Marker string Marker string
StartAfter string StartAfter string
@ -124,7 +124,7 @@ func (h *handler) listObjects(w http.ResponseWriter, r *http.Request) (*listObje
Bucket: arg.Bucket, Bucket: arg.Bucket,
Prefix: arg.Prefix, Prefix: arg.Prefix,
MaxKeys: arg.MaxKeys, MaxKeys: arg.MaxKeys,
Delimiter: arg.Delimeter, Delimiter: arg.Delimiter,
Marker: marker, Marker: marker,
}) })
if err != nil { if err != nil {
@ -170,7 +170,7 @@ func encodeV1(arg *listObjectsArgs, list *layer.ListObjectsInfo) *ListObjectsRes
Marker: arg.Marker, Marker: arg.Marker,
Prefix: arg.Prefix, Prefix: arg.Prefix,
MaxKeys: arg.MaxKeys, MaxKeys: arg.MaxKeys,
Delimiter: arg.Delimeter, Delimiter: arg.Delimiter,
IsTruncated: list.IsTruncated, IsTruncated: list.IsTruncated,
NextMarker: list.NextMarker, NextMarker: list.NextMarker,
@ -230,7 +230,7 @@ func encodeV2(arg *listObjectsArgs, list *layer.ListObjectsInfo) *ListObjectsV2R
Prefix: arg.Prefix, Prefix: arg.Prefix,
KeyCount: len(list.Objects), KeyCount: len(list.Objects),
MaxKeys: arg.MaxKeys, MaxKeys: arg.MaxKeys,
Delimiter: arg.Delimeter, Delimiter: arg.Delimiter,
StartAfter: arg.StartAfter, StartAfter: arg.StartAfter,
IsTruncated: list.IsTruncated, IsTruncated: list.IsTruncated,
@ -281,7 +281,7 @@ func parseListObjectArgs(r *http.Request) (*listObjectsArgs, error) {
res.Prefix = r.URL.Query().Get("prefix") res.Prefix = r.URL.Query().Get("prefix")
res.Marker = r.URL.Query().Get("marker") res.Marker = r.URL.Query().Get("marker")
res.Delimeter = r.URL.Query().Get("delimiter") res.Delimiter = r.URL.Query().Get("delimiter")
res.Encode = r.URL.Query().Get("encoding-type") res.Encode = r.URL.Query().Get("encoding-type")
res.StartAfter = r.URL.Query().Get("start-after") res.StartAfter = r.URL.Query().Get("start-after")
apiVersionStr := r.URL.Query().Get("list-type") apiVersionStr := r.URL.Query().Get("list-type")

View file

@ -201,7 +201,7 @@ func (n *layer) ListObjects(ctx context.Context, p *ListObjectsParams) (*ListObj
bkt *BucketInfo bkt *BucketInfo
ids []*object.ID ids []*object.ID
result ListObjectsInfo result ListObjectsInfo
uniqNames = make(map[string]struct{}) uniqNames = make(map[string]bool)
) )
if bkt, err = n.GetBucketInfo(ctx, p.Bucket); err != nil { if bkt, err = n.GetBucketInfo(ctx, p.Bucket); err != nil {
@ -250,18 +250,20 @@ func (n *layer) ListObjects(ctx context.Context, p *ListObjectsParams) (*ListObj
// and look for entities after prefix. If entity does not have any // and look for entities after prefix. If entity does not have any
// sub-entities, then it is a file, else directory. // sub-entities, then it is a file, else directory.
if oi := objectInfoFromMeta(bkt, meta, p.Prefix); oi != nil { if oi := objectInfoFromMeta(bkt, meta, p.Prefix, p.Delimiter); oi != nil {
// use only unique dir names // use only unique dir names
if _, ok := uniqNames[oi.Name]; !ok { if _, ok := uniqNames[oi.Name]; ok {
continue
}
if len(p.Marker) > 0 && oi.Name <= p.Marker { if len(p.Marker) > 0 && oi.Name <= p.Marker {
continue continue
} }
uniqNames[oi.Name] = struct{}{}
uniqNames[oi.Name] = oi.isDir
result.Objects = append(result.Objects, oi) result.Objects = append(result.Objects, oi)
} }
} }
}
sort.Slice(result.Objects, func(i, j int) bool { sort.Slice(result.Objects, func(i, j int) bool {
return result.Objects[i].Name < result.Objects[j].Name return result.Objects[i].Name < result.Objects[j].Name
@ -272,6 +274,14 @@ func (n *layer) ListObjects(ctx context.Context, p *ListObjectsParams) (*ListObj
result.Objects = result.Objects[:p.MaxKeys] result.Objects = result.Objects[:p.MaxKeys]
result.NextMarker = result.Objects[len(result.Objects)-1].Name result.NextMarker = result.Objects[len(result.Objects)-1].Name
} }
for i, oi := range result.Objects {
if isDir := uniqNames[oi.Name]; isDir {
result.Objects = append(result.Objects[:i], result.Objects[i+1:]...)
result.Prefixes = append(result.Prefixes, oi.Name)
}
}
return &result, nil return &result, nil
} }
@ -358,7 +368,7 @@ func (n *layer) GetObjectInfo(ctx context.Context, bucketName, filename string)
return nil, err return nil, err
} }
return objectInfoFromMeta(bkt, meta, rootSeparator), nil return objectInfoFromMeta(bkt, meta, "", ""), nil
} }
// PutObject into storage. // PutObject into storage.

View file

@ -55,11 +55,8 @@ type (
} }
) )
const ( // PathSeparator is a path components separator string.
rootSeparator = "root://" const PathSeparator = string(os.PathSeparator)
// PathSeparator is a path components separator string.
PathSeparator = string(os.PathSeparator)
)
func userHeaders(attrs []*object.Attribute) map[string]string { func userHeaders(attrs []*object.Attribute) map[string]string {
result := make(map[string]string, len(attrs)) result := make(map[string]string, len(attrs))
@ -71,30 +68,25 @@ func userHeaders(attrs []*object.Attribute) map[string]string {
return result return result
} }
func objectInfoFromMeta(bkt *BucketInfo, meta *object.Object, prefix string) *ObjectInfo { func objectInfoFromMeta(bkt *BucketInfo, meta *object.Object, prefix, delimiter string) *ObjectInfo {
var ( var (
isDir bool isDir bool
size int64 size int64
mimeType string mimeType string
creation time.Time creation time.Time
filename = meta.ID().String() filename = filenameFromObject(meta)
name, dirname = nameFromObject(meta)
) )
if !strings.HasPrefix(dirname, prefix) && prefix != rootSeparator { if !strings.HasPrefix(filename, prefix) {
return nil return nil
} }
if ln := len(prefix); ln > 0 && prefix[ln-1:] != PathSeparator {
prefix += PathSeparator
}
userHeaders := userHeaders(meta.Attributes()) userHeaders := userHeaders(meta.Attributes())
if val, ok := userHeaders[object.AttributeFileName]; ok {
filename = val
delete(userHeaders, object.AttributeFileName) delete(userHeaders, object.AttributeFileName)
if contentType, ok := userHeaders[object.AttributeContentType]; ok {
mimeType = contentType
delete(userHeaders, object.AttributeContentType)
} }
if val, ok := userHeaders[object.AttributeTimestamp]; !ok { if val, ok := userHeaders[object.AttributeTimestamp]; !ok {
// ignore empty value // ignore empty value
} else if dt, err := strconv.ParseInt(val, 10, 64); err == nil { } else if dt, err := strconv.ParseInt(val, 10, 64); err == nil {
@ -102,20 +94,18 @@ func objectInfoFromMeta(bkt *BucketInfo, meta *object.Object, prefix string) *Ob
delete(userHeaders, object.AttributeTimestamp) delete(userHeaders, object.AttributeTimestamp)
} }
tail := strings.TrimPrefix(dirname, prefix) if len(delimiter) > 0 {
index := strings.Index(tail, PathSeparator) tail := strings.TrimPrefix(filename, prefix)
index := strings.Index(tail, delimiter)
if prefix == rootSeparator { if index >= 0 {
size = int64(meta.PayloadSize())
mimeType = http.DetectContentType(meta.Payload())
} else if index < 0 {
filename = name
size = int64(meta.PayloadSize())
mimeType = http.DetectContentType(meta.Payload())
} else {
isDir = true isDir = true
filename = tail[:index] + PathSeparator filename = prefix + tail[:index+1]
userHeaders = nil userHeaders = nil
} else {
size, mimeType = getSizeAndMimeType(meta, mimeType)
}
} else {
size, mimeType = getSizeAndMimeType(meta, mimeType)
} }
return &ObjectInfo{ return &ObjectInfo{
@ -132,18 +122,23 @@ func objectInfoFromMeta(bkt *BucketInfo, meta *object.Object, prefix string) *Ob
} }
} }
func nameFromObject(o *object.Object) (string, string) { func getSizeAndMimeType(meta *object.Object, contentType string) (size int64, mimeType string) {
var name = o.ID().String() size = int64(meta.PayloadSize())
mimeType = contentType
if len(mimeType) == 0 {
mimeType = http.DetectContentType(meta.Payload())
}
return
}
func filenameFromObject(o *object.Object) string {
var name = o.ID().String()
for _, attr := range o.Attributes() { for _, attr := range o.Attributes() {
if attr.Key() == object.AttributeFileName { if attr.Key() == object.AttributeFileName {
name = attr.Value() return attr.Value()
break
} }
} }
return name
return NameFromString(name)
} }
// NameFromString splits name into base file name and directory path. // NameFromString splits name into base file name and directory path.

View file

@ -3,7 +3,6 @@ package layer
import ( import (
"net/http" "net/http"
"strconv" "strconv"
"strings"
"testing" "testing"
"time" "time"
@ -40,7 +39,7 @@ func newTestObject(oid *object.ID, bkt *BucketInfo, name string) *object.Object
return raw.Object() return raw.Object()
} }
func newTestInfo(oid *object.ID, bkt *BucketInfo, name, prefix string) *ObjectInfo { func newTestInfo(oid *object.ID, bkt *BucketInfo, name string, isDir bool) *ObjectInfo {
info := &ObjectInfo{ info := &ObjectInfo{
id: oid, id: oid,
Name: name, Name: name,
@ -52,34 +51,16 @@ func newTestInfo(oid *object.ID, bkt *BucketInfo, name, prefix string) *ObjectIn
Headers: make(map[string]string), Headers: make(map[string]string),
} }
if prefix == rootSeparator { if isDir {
return info
}
_, dirname := testNameFromObjectName(name)
if ln := len(prefix); ln > 0 && prefix[ln-1:] != PathSeparator {
prefix += PathSeparator
}
tail := strings.TrimPrefix(dirname, prefix)
if index := strings.Index(tail, PathSeparator); index >= 0 {
info.isDir = true info.isDir = true
info.Size = 0 info.Size = 0
info.ContentType = "" info.ContentType = ""
info.Name = tail[:index+1]
info.Headers = nil info.Headers = nil
} }
return info return info
} }
func testNameFromObjectName(name string) (string, string) {
ind := strings.LastIndex(name, PathSeparator)
return name[ind+1:], name[:ind+1]
}
func Test_objectInfoFromMeta(t *testing.T) { func Test_objectInfoFromMeta(t *testing.T) {
uid := owner.NewID() uid := owner.NewID()
oid := object.NewID() oid := object.NewID()
@ -97,95 +78,77 @@ func Test_objectInfoFromMeta(t *testing.T) {
prefix string prefix string
result *ObjectInfo result *ObjectInfo
object *object.Object object *object.Object
delimiter string
infoName string
}{ }{
{ {
name: "test.jpg", name: "small.jpg",
prefix: "", result: newTestInfo(oid, bkt, "small.jpg", false),
infoName: "test.jpg", object: newTestObject(oid, bkt, "small.jpg"),
result: newTestInfo(oid, bkt, "test.jpg", ""), },
object: newTestObject(oid, bkt, "test.jpg"), {
name: "small.jpg not matched prefix",
prefix: "big",
result: nil,
object: newTestObject(oid, bkt, "small.jpg"),
},
{
name: "small.jpg delimiter",
delimiter: "/",
result: newTestInfo(oid, bkt, "small.jpg", false),
object: newTestObject(oid, bkt, "small.jpg"),
}, },
{ {
name: "test/small.jpg", name: "test/small.jpg",
prefix: "", result: newTestInfo(oid, bkt, "test/small.jpg", false),
infoName: "test/",
result: newTestInfo(oid, bkt, "test/small.jpg", ""),
object: newTestObject(oid, bkt, "test/small.jpg"), object: newTestObject(oid, bkt, "test/small.jpg"),
}, },
{ {
name: "test/small.jpg raw", name: "test/small.jpg with prefix and delimiter",
prefix: rootSeparator, prefix: "test/",
infoName: "test/small.jpg", delimiter: "/",
result: newTestInfo(oid, bkt, "test/small.jpg", rootSeparator), result: newTestInfo(oid, bkt, "test/small.jpg", false),
object: newTestObject(oid, bkt, "test/small.jpg"), object: newTestObject(oid, bkt, "test/small.jpg"),
}, },
{ {
name: "test/a/b/c/d/e/f/g/h/small.jpg", name: "a/b/small.jpg",
prefix: "", prefix: "a",
infoName: "test/", result: newTestInfo(oid, bkt, "a/b/small.jpg", false),
result: newTestInfo(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg", ""), object: newTestObject(oid, bkt, "a/b/small.jpg"),
object: newTestObject(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg"),
}, },
{ {
name: "test/a/b/c/d/e/f/g/h/small.jpg", name: "a/b/small.jpg",
prefix: "test", prefix: "a/",
infoName: "a/", delimiter: "/",
result: newTestInfo(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg", "test"), result: newTestInfo(oid, bkt, "a/b/", true),
object: newTestObject(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg"), object: newTestObject(oid, bkt, "a/b/small.jpg"),
}, },
{ {
name: "test/a/b/c/d/e/f/g/h/small.jpg", name: "a/b/c/small.jpg",
prefix: "test/a", prefix: "a/",
infoName: "b/", delimiter: "/",
result: newTestInfo(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg", "test/a"), result: newTestInfo(oid, bkt, "a/b/", true),
object: newTestObject(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg"), object: newTestObject(oid, bkt, "a/b/c/small.jpg"),
}, },
{ {
name: "test/a/b/c/d/e/f/g/h/small.jpg", name: "a/b/c/small.jpg",
prefix: "test/a/b", prefix: "a/b/c/s",
infoName: "c/", delimiter: "/",
result: newTestInfo(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg", "test/a/b"), result: newTestInfo(oid, bkt, "a/b/c/small.jpg", false),
object: newTestObject(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg"), object: newTestObject(oid, bkt, "a/b/c/small.jpg"),
}, },
{ {
name: "test/a/b/c/d/e/f/g/h/small.jpg with slash", name: "a/b/c/big.jpg",
prefix: "test/a/b/", prefix: "a/b/",
infoName: "c/", delimiter: "/",
result: newTestInfo(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg", "test/a/b/"), result: newTestInfo(oid, bkt, "a/b/c/", true),
object: newTestObject(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg"), object: newTestObject(oid, bkt, "a/b/c/big.jpg"),
},
{
name: "test/a/b/c/d/e/f/g/h/small.jpg",
prefix: "test/a/b/c",
infoName: "d/",
result: newTestInfo(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg", "test/a/b/c"),
object: newTestObject(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg"),
},
{
name: "test/a/b/c/d/e/f/g/h/small.jpg",
prefix: "test/a/b/c/d",
infoName: "e/",
result: newTestInfo(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg", "test/a/b/c/d"),
object: newTestObject(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg"),
}, },
} }
for _, tc := range cases { for _, tc := range cases {
t.Run(tc.name+"_"+tc.infoName, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
info := objectInfoFromMeta(bkt, tc.object, tc.prefix) info := objectInfoFromMeta(bkt, tc.object, tc.prefix, tc.delimiter)
require.Equal(t, tc.result, info) require.Equal(t, tc.result, info)
require.Equal(t, tc.infoName, info.Name)
}) })
} }
} }