TrueCloudLab/frostfs-s3-gw
17
3
Fork 19
Code Issues 28 Pull requests 7 Projects Releases 18 Packages Activity Actions

Merge pull request #115 from KirillovDenis/feature/98-support_delimiter

Browse source 
[#98] Supported listObjects delimiter
This commit is contained in:
Angira Kekteeva 2021-06-30 12:43:10 +03:00 committed by GitHub
commit d2bed40c19
4 changed files with 113 additions and 145 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
api/handler/list.go
View file

@ -14,7 +14,7 @@ import (
type listObjectsArgs struct {
Bucket string
Delimeter string
Delimiter string
Encode string
Marker string
StartAfter string
@ -124,7 +124,7 @@ func (h *handler) listObjects(w http.ResponseWriter, r *http.Request) (*listObje
Bucket: arg.Bucket,
Prefix: arg.Prefix,
MaxKeys: arg.MaxKeys,
Delimiter: arg.Delimeter,
Delimiter: arg.Delimiter,
Marker: marker,
})
if err != nil {
@ -170,7 +170,7 @@ func encodeV1(arg *listObjectsArgs, list *layer.ListObjectsInfo) *ListObjectsRes
Marker: arg.Marker,
Prefix: arg.Prefix,
MaxKeys: arg.MaxKeys,
Delimiter: arg.Delimeter,
Delimiter: arg.Delimiter,
IsTruncated: list.IsTruncated,
NextMarker: list.NextMarker,
@ -230,7 +230,7 @@ func encodeV2(arg *listObjectsArgs, list *layer.ListObjectsInfo) *ListObjectsV2R
Prefix: arg.Prefix,
KeyCount: len(list.Objects),
MaxKeys: arg.MaxKeys,
Delimiter: arg.Delimeter,
Delimiter: arg.Delimiter,
StartAfter: arg.StartAfter,
IsTruncated: list.IsTruncated,
@ -281,7 +281,7 @@ func parseListObjectArgs(r *http.Request) (*listObjectsArgs, error) {
res.Prefix = r.URL.Query().Get("prefix")
res.Marker = r.URL.Query().Get("marker")
res.Delimeter = r.URL.Query().Get("delimiter")
res.Delimiter = r.URL.Query().Get("delimiter")
res.Encode = r.URL.Query().Get("encoding-type")
res.StartAfter = r.URL.Query().Get("start-after")
apiVersionStr := r.URL.Query().Get("list-type")

30
api/layer/layer.go
View file

@ -201,7 +201,7 @@ func (n *layer) ListObjects(ctx context.Context, p *ListObjectsParams) (*ListObj
bkt *BucketInfo
ids []*object.ID
result ListObjectsInfo
uniqNames = make(map[string]struct{})
uniqNames = make(map[string]bool)
)
if bkt, err = n.GetBucketInfo(ctx, p.Bucket); err != nil {
@ -250,16 +250,18 @@ func (n *layer) ListObjects(ctx context.Context, p *ListObjectsParams) (*ListObj
// and look for entities after prefix. If entity does not have any
// sub-entities, then it is a file, else directory.
if oi := objectInfoFromMeta(bkt, meta, p.Prefix); oi != nil {
if oi := objectInfoFromMeta(bkt, meta, p.Prefix, p.Delimiter); oi != nil {
// use only unique dir names
if _, ok := uniqNames[oi.Name]; !ok {
if len(p.Marker) > 0 && oi.Name <= p.Marker {
continue
}
uniqNames[oi.Name] = struct{}{}
result.Objects = append(result.Objects, oi)
if _, ok := uniqNames[oi.Name]; ok {
continue
}
if len(p.Marker) > 0 && oi.Name <= p.Marker {
continue
}
uniqNames[oi.Name] = oi.isDir
result.Objects = append(result.Objects, oi)
}
}
@ -272,6 +274,14 @@ func (n *layer) ListObjects(ctx context.Context, p *ListObjectsParams) (*ListObj
result.Objects = result.Objects[:p.MaxKeys]
result.NextMarker = result.Objects[len(result.Objects)-1].Name
}
for i, oi := range result.Objects {
if isDir := uniqNames[oi.Name]; isDir {
result.Objects = append(result.Objects[:i], result.Objects[i+1:]...)
result.Prefixes = append(result.Prefixes, oi.Name)
}
}
return &result, nil
}
@ -358,7 +368,7 @@ func (n *layer) GetObjectInfo(ctx context.Context, bucketName, filename string)
return nil, err
}
return objectInfoFromMeta(bkt, meta, rootSeparator), nil
return objectInfoFromMeta(bkt, meta, "", ""), nil
}
// PutObject into storage.

77
api/layer/util.go
View file

@ -55,11 +55,8 @@ type (
}
)
const (
rootSeparator = "root://"
// PathSeparator is a path components separator string.
PathSeparator = string(os.PathSeparator)
)
// PathSeparator is a path components separator string.
const PathSeparator = string(os.PathSeparator)
func userHeaders(attrs []*object.Attribute) map[string]string {
result := make(map[string]string, len(attrs))
@ -71,30 +68,25 @@ func userHeaders(attrs []*object.Attribute) map[string]string {
return result
}
func objectInfoFromMeta(bkt *BucketInfo, meta *object.Object, prefix string) *ObjectInfo {
func objectInfoFromMeta(bkt *BucketInfo, meta *object.Object, prefix, delimiter string) *ObjectInfo {
var (
isDir bool
size int64
mimeType string
creation time.Time
filename = meta.ID().String()
name, dirname = nameFromObject(meta)
isDir bool
size int64
mimeType string
creation time.Time
filename = filenameFromObject(meta)
)
if !strings.HasPrefix(dirname, prefix) && prefix != rootSeparator {
if !strings.HasPrefix(filename, prefix) {
return nil
}
if ln := len(prefix); ln > 0 && prefix[ln-1:] != PathSeparator {
prefix += PathSeparator
}
userHeaders := userHeaders(meta.Attributes())
if val, ok := userHeaders[object.AttributeFileName]; ok {
filename = val
delete(userHeaders, object.AttributeFileName)
delete(userHeaders, object.AttributeFileName)
if contentType, ok := userHeaders[object.AttributeContentType]; ok {
mimeType = contentType
delete(userHeaders, object.AttributeContentType)
}
if val, ok := userHeaders[object.AttributeTimestamp]; !ok {
// ignore empty value
} else if dt, err := strconv.ParseInt(val, 10, 64); err == nil {
@ -102,20 +94,18 @@ func objectInfoFromMeta(bkt *BucketInfo, meta *object.Object, prefix string) *Ob
delete(userHeaders, object.AttributeTimestamp)
}
tail := strings.TrimPrefix(dirname, prefix)
index := strings.Index(tail, PathSeparator)
if prefix == rootSeparator {
size = int64(meta.PayloadSize())
mimeType = http.DetectContentType(meta.Payload())
} else if index < 0 {
filename = name
size = int64(meta.PayloadSize())
mimeType = http.DetectContentType(meta.Payload())
if len(delimiter) > 0 {
tail := strings.TrimPrefix(filename, prefix)
index := strings.Index(tail, delimiter)
if index >= 0 {
isDir = true
filename = prefix + tail[:index+1]
userHeaders = nil
} else {
size, mimeType = getSizeAndMimeType(meta, mimeType)
}
} else {
isDir = true
filename = tail[:index] + PathSeparator
userHeaders = nil
size, mimeType = getSizeAndMimeType(meta, mimeType)
}
return &ObjectInfo{
@ -132,18 +122,23 @@ func objectInfoFromMeta(bkt *BucketInfo, meta *object.Object, prefix string) *Ob
}
}
func nameFromObject(o *object.Object) (string, string) {
var name = o.ID().String()
func getSizeAndMimeType(meta *object.Object, contentType string) (size int64, mimeType string) {
size = int64(meta.PayloadSize())
mimeType = contentType
if len(mimeType) == 0 {
mimeType = http.DetectContentType(meta.Payload())
}
return
}
func filenameFromObject(o *object.Object) string {
var name = o.ID().String()
for _, attr := range o.Attributes() {
if attr.Key() == object.AttributeFileName {
name = attr.Value()
break
return attr.Value()
}
}
return NameFromString(name)
return name
}
// NameFromString splits name into base file name and directory path.

141
api/layer/util_test.go
View file

@ -3,7 +3,6 @@ package layer
import (
"net/http"
"strconv"
"strings"
"testing"
"time"
@ -40,7 +39,7 @@ func newTestObject(oid *object.ID, bkt *BucketInfo, name string) *object.Object
return raw.Object()
}
func newTestInfo(oid *object.ID, bkt *BucketInfo, name, prefix string) *ObjectInfo {
func newTestInfo(oid *object.ID, bkt *BucketInfo, name string, isDir bool) *ObjectInfo {
info := &ObjectInfo{
id: oid,
Name: name,
@ -52,34 +51,16 @@ func newTestInfo(oid *object.ID, bkt *BucketInfo, name, prefix string) *ObjectIn
Headers: make(map[string]string),
}
if prefix == rootSeparator {
return info
}
_, dirname := testNameFromObjectName(name)
if ln := len(prefix); ln > 0 && prefix[ln-1:] != PathSeparator {
prefix += PathSeparator
}
tail := strings.TrimPrefix(dirname, prefix)
if index := strings.Index(tail, PathSeparator); index >= 0 {
if isDir {
info.isDir = true
info.Size = 0
info.ContentType = ""
info.Name = tail[:index+1]
info.Headers = nil
}
return info
}
func testNameFromObjectName(name string) (string, string) {
ind := strings.LastIndex(name, PathSeparator)
return name[ind+1:], name[:ind+1]
}
func Test_objectInfoFromMeta(t *testing.T) {
uid := owner.NewID()
oid := object.NewID()
@ -93,99 +74,81 @@ func Test_objectInfoFromMeta(t *testing.T) {
}
cases := []struct {
name string
prefix string
result *ObjectInfo
object *object.Object
infoName string
name string
prefix string
result *ObjectInfo
object *object.Object
delimiter string
}{
{
name: "test.jpg",
prefix: "",
infoName: "test.jpg",
result: newTestInfo(oid, bkt, "test.jpg", ""),
object: newTestObject(oid, bkt, "test.jpg"),
name: "small.jpg",
result: newTestInfo(oid, bkt, "small.jpg", false),
object: newTestObject(oid, bkt, "small.jpg"),
},
{
name: "test/small.jpg",
prefix: "",
infoName: "test/",
result: newTestInfo(oid, bkt, "test/small.jpg", ""),
object: newTestObject(oid, bkt, "test/small.jpg"),
name: "small.jpg not matched prefix",
prefix: "big",
result: nil,
object: newTestObject(oid, bkt, "small.jpg"),
},
{
name: "test/small.jpg raw",
prefix: rootSeparator,
infoName: "test/small.jpg",
result: newTestInfo(oid, bkt, "test/small.jpg", rootSeparator),
object: newTestObject(oid, bkt, "test/small.jpg"),
name: "small.jpg delimiter",
delimiter: "/",
result: newTestInfo(oid, bkt, "small.jpg", false),
object: newTestObject(oid, bkt, "small.jpg"),
},
{
name: "test/a/b/c/d/e/f/g/h/small.jpg",
prefix: "",
infoName: "test/",
result: newTestInfo(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg", ""),
object: newTestObject(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg"),
name: "test/small.jpg",
result: newTestInfo(oid, bkt, "test/small.jpg", false),
object: newTestObject(oid, bkt, "test/small.jpg"),
},
{
name: "test/a/b/c/d/e/f/g/h/small.jpg",
prefix: "test",
infoName: "a/",
result: newTestInfo(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg", "test"),
object: newTestObject(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg"),
name: "test/small.jpg with prefix and delimiter",
prefix: "test/",
delimiter: "/",
result: newTestInfo(oid, bkt, "test/small.jpg", false),
object: newTestObject(oid, bkt, "test/small.jpg"),
},
{
name: "test/a/b/c/d/e/f/g/h/small.jpg",
prefix: "test/a",
infoName: "b/",
result: newTestInfo(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg", "test/a"),
object: newTestObject(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg"),
name: "a/b/small.jpg",
prefix: "a",
result: newTestInfo(oid, bkt, "a/b/small.jpg", false),
object: newTestObject(oid, bkt, "a/b/small.jpg"),
},
{
name: "test/a/b/c/d/e/f/g/h/small.jpg",
prefix: "test/a/b",
infoName: "c/",
result: newTestInfo(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg", "test/a/b"),
object: newTestObject(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg"),
name: "a/b/small.jpg",
prefix: "a/",
delimiter: "/",
result: newTestInfo(oid, bkt, "a/b/", true),
object: newTestObject(oid, bkt, "a/b/small.jpg"),
},
{
name: "test/a/b/c/d/e/f/g/h/small.jpg with slash",
prefix: "test/a/b/",
infoName: "c/",
result: newTestInfo(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg", "test/a/b/"),
object: newTestObject(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg"),
name: "a/b/c/small.jpg",
prefix: "a/",
delimiter: "/",
result: newTestInfo(oid, bkt, "a/b/", true),
object: newTestObject(oid, bkt, "a/b/c/small.jpg"),
},
{
name: "test/a/b/c/d/e/f/g/h/small.jpg",
prefix: "test/a/b/c",
infoName: "d/",
result: newTestInfo(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg", "test/a/b/c"),
object: newTestObject(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg"),
name: "a/b/c/small.jpg",
prefix: "a/b/c/s",
delimiter: "/",
result: newTestInfo(oid, bkt, "a/b/c/small.jpg", false),
object: newTestObject(oid, bkt, "a/b/c/small.jpg"),
},
{
name: "test/a/b/c/d/e/f/g/h/small.jpg",
prefix: "test/a/b/c/d",
infoName: "e/",
result: newTestInfo(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg", "test/a/b/c/d"),
object: newTestObject(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg"),
name: "a/b/c/big.jpg",
prefix: "a/b/",
delimiter: "/",
result: newTestInfo(oid, bkt, "a/b/c/", true),
object: newTestObject(oid, bkt, "a/b/c/big.jpg"),
},
}
for _, tc := range cases {
t.Run(tc.name+"_"+tc.infoName, func(t *testing.T) {
info := objectInfoFromMeta(bkt, tc.object, tc.prefix)
t.Run(tc.name, func(t *testing.T) {
info := objectInfoFromMeta(bkt, tc.object, tc.prefix, tc.delimiter)
require.Equal(t, tc.result, info)
require.Equal(t, tc.infoName, info.Name)
})
}
}