From 65be38831c5f04aa083e9874324c830f8b02b2f4 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Tue, 29 Jun 2021 12:59:33 +0300 Subject: [PATCH] [#98] Supported listObjects delimiter Signed-off-by: Denis Kirillov --- api/handler/list.go | 10 +-- api/layer/layer.go | 30 ++++++--- api/layer/util.go | 77 +++++++++++----------- api/layer/util_test.go | 141 +++++++++++++++-------------------------- 4 files changed, 113 insertions(+), 145 deletions(-) diff --git a/api/handler/list.go b/api/handler/list.go index 35e74a797..a05257fb2 100644 --- a/api/handler/list.go +++ b/api/handler/list.go @@ -14,7 +14,7 @@ import ( type listObjectsArgs struct { Bucket string - Delimeter string + Delimiter string Encode string Marker string StartAfter string @@ -124,7 +124,7 @@ func (h *handler) listObjects(w http.ResponseWriter, r *http.Request) (*listObje Bucket: arg.Bucket, Prefix: arg.Prefix, MaxKeys: arg.MaxKeys, - Delimiter: arg.Delimeter, + Delimiter: arg.Delimiter, Marker: marker, }) if err != nil { @@ -170,7 +170,7 @@ func encodeV1(arg *listObjectsArgs, list *layer.ListObjectsInfo) *ListObjectsRes Marker: arg.Marker, Prefix: arg.Prefix, MaxKeys: arg.MaxKeys, - Delimiter: arg.Delimeter, + Delimiter: arg.Delimiter, IsTruncated: list.IsTruncated, NextMarker: list.NextMarker, @@ -230,7 +230,7 @@ func encodeV2(arg *listObjectsArgs, list *layer.ListObjectsInfo) *ListObjectsV2R Prefix: arg.Prefix, KeyCount: len(list.Objects), MaxKeys: arg.MaxKeys, - Delimiter: arg.Delimeter, + Delimiter: arg.Delimiter, StartAfter: arg.StartAfter, IsTruncated: list.IsTruncated, @@ -281,7 +281,7 @@ func parseListObjectArgs(r *http.Request) (*listObjectsArgs, error) { res.Prefix = r.URL.Query().Get("prefix") res.Marker = r.URL.Query().Get("marker") - res.Delimeter = r.URL.Query().Get("delimiter") + res.Delimiter = r.URL.Query().Get("delimiter") res.Encode = r.URL.Query().Get("encoding-type") res.StartAfter = r.URL.Query().Get("start-after") apiVersionStr := r.URL.Query().Get("list-type") diff --git a/api/layer/layer.go b/api/layer/layer.go index a81ba9237..30593e503 100644 --- a/api/layer/layer.go +++ b/api/layer/layer.go @@ -201,7 +201,7 @@ func (n *layer) ListObjects(ctx context.Context, p *ListObjectsParams) (*ListObj bkt *BucketInfo ids []*object.ID result ListObjectsInfo - uniqNames = make(map[string]struct{}) + uniqNames = make(map[string]bool) ) if bkt, err = n.GetBucketInfo(ctx, p.Bucket); err != nil { @@ -250,16 +250,18 @@ func (n *layer) ListObjects(ctx context.Context, p *ListObjectsParams) (*ListObj // and look for entities after prefix. If entity does not have any // sub-entities, then it is a file, else directory. - if oi := objectInfoFromMeta(bkt, meta, p.Prefix); oi != nil { + if oi := objectInfoFromMeta(bkt, meta, p.Prefix, p.Delimiter); oi != nil { // use only unique dir names - if _, ok := uniqNames[oi.Name]; !ok { - if len(p.Marker) > 0 && oi.Name <= p.Marker { - continue - } - uniqNames[oi.Name] = struct{}{} - - result.Objects = append(result.Objects, oi) + if _, ok := uniqNames[oi.Name]; ok { + continue } + if len(p.Marker) > 0 && oi.Name <= p.Marker { + continue + } + + uniqNames[oi.Name] = oi.isDir + + result.Objects = append(result.Objects, oi) } } @@ -272,6 +274,14 @@ func (n *layer) ListObjects(ctx context.Context, p *ListObjectsParams) (*ListObj result.Objects = result.Objects[:p.MaxKeys] result.NextMarker = result.Objects[len(result.Objects)-1].Name } + + for i, oi := range result.Objects { + if isDir := uniqNames[oi.Name]; isDir { + result.Objects = append(result.Objects[:i], result.Objects[i+1:]...) + result.Prefixes = append(result.Prefixes, oi.Name) + } + } + return &result, nil } @@ -358,7 +368,7 @@ func (n *layer) GetObjectInfo(ctx context.Context, bucketName, filename string) return nil, err } - return objectInfoFromMeta(bkt, meta, rootSeparator), nil + return objectInfoFromMeta(bkt, meta, "", ""), nil } // PutObject into storage. diff --git a/api/layer/util.go b/api/layer/util.go index ba60cc7ce..90f9ddb4f 100644 --- a/api/layer/util.go +++ b/api/layer/util.go @@ -55,11 +55,8 @@ type ( } ) -const ( - rootSeparator = "root://" - // PathSeparator is a path components separator string. - PathSeparator = string(os.PathSeparator) -) +// PathSeparator is a path components separator string. +const PathSeparator = string(os.PathSeparator) func userHeaders(attrs []*object.Attribute) map[string]string { result := make(map[string]string, len(attrs)) @@ -71,30 +68,25 @@ func userHeaders(attrs []*object.Attribute) map[string]string { return result } -func objectInfoFromMeta(bkt *BucketInfo, meta *object.Object, prefix string) *ObjectInfo { +func objectInfoFromMeta(bkt *BucketInfo, meta *object.Object, prefix, delimiter string) *ObjectInfo { var ( - isDir bool - size int64 - mimeType string - creation time.Time - filename = meta.ID().String() - name, dirname = nameFromObject(meta) + isDir bool + size int64 + mimeType string + creation time.Time + filename = filenameFromObject(meta) ) - if !strings.HasPrefix(dirname, prefix) && prefix != rootSeparator { + if !strings.HasPrefix(filename, prefix) { return nil } - if ln := len(prefix); ln > 0 && prefix[ln-1:] != PathSeparator { - prefix += PathSeparator - } - userHeaders := userHeaders(meta.Attributes()) - if val, ok := userHeaders[object.AttributeFileName]; ok { - filename = val - delete(userHeaders, object.AttributeFileName) + delete(userHeaders, object.AttributeFileName) + if contentType, ok := userHeaders[object.AttributeContentType]; ok { + mimeType = contentType + delete(userHeaders, object.AttributeContentType) } - if val, ok := userHeaders[object.AttributeTimestamp]; !ok { // ignore empty value } else if dt, err := strconv.ParseInt(val, 10, 64); err == nil { @@ -102,20 +94,18 @@ func objectInfoFromMeta(bkt *BucketInfo, meta *object.Object, prefix string) *Ob delete(userHeaders, object.AttributeTimestamp) } - tail := strings.TrimPrefix(dirname, prefix) - index := strings.Index(tail, PathSeparator) - - if prefix == rootSeparator { - size = int64(meta.PayloadSize()) - mimeType = http.DetectContentType(meta.Payload()) - } else if index < 0 { - filename = name - size = int64(meta.PayloadSize()) - mimeType = http.DetectContentType(meta.Payload()) + if len(delimiter) > 0 { + tail := strings.TrimPrefix(filename, prefix) + index := strings.Index(tail, delimiter) + if index >= 0 { + isDir = true + filename = prefix + tail[:index+1] + userHeaders = nil + } else { + size, mimeType = getSizeAndMimeType(meta, mimeType) + } } else { - isDir = true - filename = tail[:index] + PathSeparator - userHeaders = nil + size, mimeType = getSizeAndMimeType(meta, mimeType) } return &ObjectInfo{ @@ -132,18 +122,23 @@ func objectInfoFromMeta(bkt *BucketInfo, meta *object.Object, prefix string) *Ob } } -func nameFromObject(o *object.Object) (string, string) { - var name = o.ID().String() +func getSizeAndMimeType(meta *object.Object, contentType string) (size int64, mimeType string) { + size = int64(meta.PayloadSize()) + mimeType = contentType + if len(mimeType) == 0 { + mimeType = http.DetectContentType(meta.Payload()) + } + return +} +func filenameFromObject(o *object.Object) string { + var name = o.ID().String() for _, attr := range o.Attributes() { if attr.Key() == object.AttributeFileName { - name = attr.Value() - - break + return attr.Value() } } - - return NameFromString(name) + return name } // NameFromString splits name into base file name and directory path. diff --git a/api/layer/util_test.go b/api/layer/util_test.go index 151b8f7c1..71ad13b2e 100644 --- a/api/layer/util_test.go +++ b/api/layer/util_test.go @@ -3,7 +3,6 @@ package layer import ( "net/http" "strconv" - "strings" "testing" "time" @@ -40,7 +39,7 @@ func newTestObject(oid *object.ID, bkt *BucketInfo, name string) *object.Object return raw.Object() } -func newTestInfo(oid *object.ID, bkt *BucketInfo, name, prefix string) *ObjectInfo { +func newTestInfo(oid *object.ID, bkt *BucketInfo, name string, isDir bool) *ObjectInfo { info := &ObjectInfo{ id: oid, Name: name, @@ -52,34 +51,16 @@ func newTestInfo(oid *object.ID, bkt *BucketInfo, name, prefix string) *ObjectIn Headers: make(map[string]string), } - if prefix == rootSeparator { - return info - } - - _, dirname := testNameFromObjectName(name) - if ln := len(prefix); ln > 0 && prefix[ln-1:] != PathSeparator { - prefix += PathSeparator - } - - tail := strings.TrimPrefix(dirname, prefix) - if index := strings.Index(tail, PathSeparator); index >= 0 { + if isDir { info.isDir = true - info.Size = 0 info.ContentType = "" - info.Name = tail[:index+1] info.Headers = nil } return info } -func testNameFromObjectName(name string) (string, string) { - ind := strings.LastIndex(name, PathSeparator) - - return name[ind+1:], name[:ind+1] -} - func Test_objectInfoFromMeta(t *testing.T) { uid := owner.NewID() oid := object.NewID() @@ -93,99 +74,81 @@ func Test_objectInfoFromMeta(t *testing.T) { } cases := []struct { - name string - prefix string - result *ObjectInfo - object *object.Object - - infoName string + name string + prefix string + result *ObjectInfo + object *object.Object + delimiter string }{ { - name: "test.jpg", - prefix: "", - infoName: "test.jpg", - result: newTestInfo(oid, bkt, "test.jpg", ""), - object: newTestObject(oid, bkt, "test.jpg"), + name: "small.jpg", + result: newTestInfo(oid, bkt, "small.jpg", false), + object: newTestObject(oid, bkt, "small.jpg"), }, - { - name: "test/small.jpg", - prefix: "", - infoName: "test/", - result: newTestInfo(oid, bkt, "test/small.jpg", ""), - object: newTestObject(oid, bkt, "test/small.jpg"), + name: "small.jpg not matched prefix", + prefix: "big", + result: nil, + object: newTestObject(oid, bkt, "small.jpg"), }, - { - name: "test/small.jpg raw", - prefix: rootSeparator, - infoName: "test/small.jpg", - result: newTestInfo(oid, bkt, "test/small.jpg", rootSeparator), - object: newTestObject(oid, bkt, "test/small.jpg"), + name: "small.jpg delimiter", + delimiter: "/", + result: newTestInfo(oid, bkt, "small.jpg", false), + object: newTestObject(oid, bkt, "small.jpg"), }, - { - name: "test/a/b/c/d/e/f/g/h/small.jpg", - prefix: "", - infoName: "test/", - result: newTestInfo(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg", ""), - object: newTestObject(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg"), + name: "test/small.jpg", + result: newTestInfo(oid, bkt, "test/small.jpg", false), + object: newTestObject(oid, bkt, "test/small.jpg"), }, - { - name: "test/a/b/c/d/e/f/g/h/small.jpg", - prefix: "test", - infoName: "a/", - result: newTestInfo(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg", "test"), - object: newTestObject(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg"), + name: "test/small.jpg with prefix and delimiter", + prefix: "test/", + delimiter: "/", + result: newTestInfo(oid, bkt, "test/small.jpg", false), + object: newTestObject(oid, bkt, "test/small.jpg"), }, - { - name: "test/a/b/c/d/e/f/g/h/small.jpg", - prefix: "test/a", - infoName: "b/", - result: newTestInfo(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg", "test/a"), - object: newTestObject(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg"), + name: "a/b/small.jpg", + prefix: "a", + result: newTestInfo(oid, bkt, "a/b/small.jpg", false), + object: newTestObject(oid, bkt, "a/b/small.jpg"), }, - { - name: "test/a/b/c/d/e/f/g/h/small.jpg", - prefix: "test/a/b", - infoName: "c/", - result: newTestInfo(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg", "test/a/b"), - object: newTestObject(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg"), + name: "a/b/small.jpg", + prefix: "a/", + delimiter: "/", + result: newTestInfo(oid, bkt, "a/b/", true), + object: newTestObject(oid, bkt, "a/b/small.jpg"), }, - { - name: "test/a/b/c/d/e/f/g/h/small.jpg with slash", - prefix: "test/a/b/", - infoName: "c/", - result: newTestInfo(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg", "test/a/b/"), - object: newTestObject(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg"), + name: "a/b/c/small.jpg", + prefix: "a/", + delimiter: "/", + result: newTestInfo(oid, bkt, "a/b/", true), + object: newTestObject(oid, bkt, "a/b/c/small.jpg"), }, - { - name: "test/a/b/c/d/e/f/g/h/small.jpg", - prefix: "test/a/b/c", - infoName: "d/", - result: newTestInfo(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg", "test/a/b/c"), - object: newTestObject(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg"), + name: "a/b/c/small.jpg", + prefix: "a/b/c/s", + delimiter: "/", + result: newTestInfo(oid, bkt, "a/b/c/small.jpg", false), + object: newTestObject(oid, bkt, "a/b/c/small.jpg"), }, - { - name: "test/a/b/c/d/e/f/g/h/small.jpg", - prefix: "test/a/b/c/d", - infoName: "e/", - result: newTestInfo(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg", "test/a/b/c/d"), - object: newTestObject(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg"), + name: "a/b/c/big.jpg", + prefix: "a/b/", + delimiter: "/", + result: newTestInfo(oid, bkt, "a/b/c/", true), + object: newTestObject(oid, bkt, "a/b/c/big.jpg"), }, } for _, tc := range cases { - t.Run(tc.name+"_"+tc.infoName, func(t *testing.T) { - info := objectInfoFromMeta(bkt, tc.object, tc.prefix) + t.Run(tc.name, func(t *testing.T) { + info := objectInfoFromMeta(bkt, tc.object, tc.prefix, tc.delimiter) require.Equal(t, tc.result, info) - require.Equal(t, tc.infoName, info.Name) }) } }