Merge pull request #115 from KirillovDenis/feature/98-support_delimiter

[#98] Supported listObjects delimiter

api/handler/list.go

@@ -14,7 +14,7 @@ import (
 
 type listObjectsArgs struct {
 	Bucket     string
-	Delimeter  string
+	Delimiter  string
 	Encode     string
 	Marker     string
 	StartAfter string
@@ -124,7 +124,7 @@ func (h *handler) listObjects(w http.ResponseWriter, r *http.Request) (*listObje
 		Bucket:    arg.Bucket,
 		Prefix:    arg.Prefix,
 		MaxKeys:   arg.MaxKeys,
-		Delimiter: arg.Delimeter,
+		Delimiter: arg.Delimiter,
 		Marker:    marker,
 	})
 	if err != nil {
@@ -170,7 +170,7 @@ func encodeV1(arg *listObjectsArgs, list *layer.ListObjectsInfo) *ListObjectsRes
 		Marker:       arg.Marker,
 		Prefix:       arg.Prefix,
 		MaxKeys:      arg.MaxKeys,
-		Delimiter:    arg.Delimeter,
+		Delimiter:    arg.Delimiter,
 
 		IsTruncated: list.IsTruncated,
 		NextMarker:  list.NextMarker,
@@ -230,7 +230,7 @@ func encodeV2(arg *listObjectsArgs, list *layer.ListObjectsInfo) *ListObjectsV2R
 		Prefix:       arg.Prefix,
 		KeyCount:     len(list.Objects),
 		MaxKeys:      arg.MaxKeys,
-		Delimiter:    arg.Delimeter,
+		Delimiter:    arg.Delimiter,
 		StartAfter:   arg.StartAfter,
 
 		IsTruncated: list.IsTruncated,
@@ -281,7 +281,7 @@ func parseListObjectArgs(r *http.Request) (*listObjectsArgs, error) {
 
 	res.Prefix = r.URL.Query().Get("prefix")
 	res.Marker = r.URL.Query().Get("marker")
-	res.Delimeter = r.URL.Query().Get("delimiter")
+	res.Delimiter = r.URL.Query().Get("delimiter")
 	res.Encode = r.URL.Query().Get("encoding-type")
 	res.StartAfter = r.URL.Query().Get("start-after")
 	apiVersionStr := r.URL.Query().Get("list-type")

api/layer/layer.go

@@ -201,7 +201,7 @@ func (n *layer) ListObjects(ctx context.Context, p *ListObjectsParams) (*ListObj
 		bkt       *BucketInfo
 		ids       []*object.ID
 		result    ListObjectsInfo
-		uniqNames = make(map[string]struct{})
+		uniqNames = make(map[string]bool)
 	)
 
 	if bkt, err = n.GetBucketInfo(ctx, p.Bucket); err != nil {
@@ -250,18 +250,20 @@ func (n *layer) ListObjects(ctx context.Context, p *ListObjectsParams) (*ListObj
 		// and look for entities after prefix. If entity does not have any
 		// sub-entities, then it is a file, else directory.
 
-		if oi := objectInfoFromMeta(bkt, meta, p.Prefix); oi != nil {
+		if oi := objectInfoFromMeta(bkt, meta, p.Prefix, p.Delimiter); oi != nil {
 			// use only unique dir names
-			if _, ok := uniqNames[oi.Name]; !ok {
+			if _, ok := uniqNames[oi.Name]; ok {
+				continue
+			}
 			if len(p.Marker) > 0 && oi.Name <= p.Marker {
 				continue
 			}
-				uniqNames[oi.Name] = struct{}{}
+
+			uniqNames[oi.Name] = oi.isDir
 
 			result.Objects = append(result.Objects, oi)
 		}
 	}
-	}
 
 	sort.Slice(result.Objects, func(i, j int) bool {
 		return result.Objects[i].Name < result.Objects[j].Name
@@ -272,6 +274,14 @@ func (n *layer) ListObjects(ctx context.Context, p *ListObjectsParams) (*ListObj
 		result.Objects = result.Objects[:p.MaxKeys]
 		result.NextMarker = result.Objects[len(result.Objects)-1].Name
 	}
+
+	for i, oi := range result.Objects {
+		if isDir := uniqNames[oi.Name]; isDir {
+			result.Objects = append(result.Objects[:i], result.Objects[i+1:]...)
+			result.Prefixes = append(result.Prefixes, oi.Name)
+		}
+	}
+
 	return &result, nil
 }
 
@@ -358,7 +368,7 @@ func (n *layer) GetObjectInfo(ctx context.Context, bucketName, filename string)
 		return nil, err
 	}
 
-	return objectInfoFromMeta(bkt, meta, rootSeparator), nil
+	return objectInfoFromMeta(bkt, meta, "", ""), nil
 }
 
 // PutObject into storage.

api/layer/util.go

@@ -55,11 +55,8 @@ type (
 	}
 )
 
-const (
+// PathSeparator is a path components separator string.
-	rootSeparator = "root://"
+const PathSeparator = string(os.PathSeparator)
-	// PathSeparator is a path components separator string.
-	PathSeparator = string(os.PathSeparator)
-)
 
 func userHeaders(attrs []*object.Attribute) map[string]string {
 	result := make(map[string]string, len(attrs))
@@ -71,30 +68,25 @@ func userHeaders(attrs []*object.Attribute) map[string]string {
 	return result
 }
 
-func objectInfoFromMeta(bkt *BucketInfo, meta *object.Object, prefix string) *ObjectInfo {
+func objectInfoFromMeta(bkt *BucketInfo, meta *object.Object, prefix, delimiter string) *ObjectInfo {
 	var (
 		isDir    bool
 		size     int64
 		mimeType string
 		creation time.Time
-		filename      = meta.ID().String()
+		filename = filenameFromObject(meta)
-		name, dirname = nameFromObject(meta)
 	)
 
-	if !strings.HasPrefix(dirname, prefix) && prefix != rootSeparator {
+	if !strings.HasPrefix(filename, prefix) {
 		return nil
 	}
 
-	if ln := len(prefix); ln > 0 && prefix[ln-1:] != PathSeparator {
-		prefix += PathSeparator
-	}
-
 	userHeaders := userHeaders(meta.Attributes())
-	if val, ok := userHeaders[object.AttributeFileName]; ok {
-		filename = val
 	delete(userHeaders, object.AttributeFileName)
+	if contentType, ok := userHeaders[object.AttributeContentType]; ok {
+		mimeType = contentType
+		delete(userHeaders, object.AttributeContentType)
 	}
-
 	if val, ok := userHeaders[object.AttributeTimestamp]; !ok {
 		// ignore empty value
 	} else if dt, err := strconv.ParseInt(val, 10, 64); err == nil {
@@ -102,20 +94,18 @@ func objectInfoFromMeta(bkt *BucketInfo, meta *object.Object, prefix string) *Ob
 		delete(userHeaders, object.AttributeTimestamp)
 	}
 
-	tail := strings.TrimPrefix(dirname, prefix)
+	if len(delimiter) > 0 {
-	index := strings.Index(tail, PathSeparator)
+		tail := strings.TrimPrefix(filename, prefix)
-
+		index := strings.Index(tail, delimiter)
-	if prefix == rootSeparator {
+		if index >= 0 {
-		size = int64(meta.PayloadSize())
-		mimeType = http.DetectContentType(meta.Payload())
-	} else if index < 0 {
-		filename = name
-		size = int64(meta.PayloadSize())
-		mimeType = http.DetectContentType(meta.Payload())
-	} else {
 			isDir = true
-		filename = tail[:index] + PathSeparator
+			filename = prefix + tail[:index+1]
 			userHeaders = nil
+		} else {
+			size, mimeType = getSizeAndMimeType(meta, mimeType)
+		}
+	} else {
+		size, mimeType = getSizeAndMimeType(meta, mimeType)
 	}
 
 	return &ObjectInfo{
@@ -132,18 +122,23 @@ func objectInfoFromMeta(bkt *BucketInfo, meta *object.Object, prefix string) *Ob
 	}
 }
 
-func nameFromObject(o *object.Object) (string, string) {
+func getSizeAndMimeType(meta *object.Object, contentType string) (size int64, mimeType string) {
-	var name = o.ID().String()
+	size = int64(meta.PayloadSize())
+	mimeType = contentType
+	if len(mimeType) == 0 {
+		mimeType = http.DetectContentType(meta.Payload())
+	}
+	return
+}
 
+func filenameFromObject(o *object.Object) string {
+	var name = o.ID().String()
 	for _, attr := range o.Attributes() {
 		if attr.Key() == object.AttributeFileName {
-			name = attr.Value()
+			return attr.Value()
-
-			break
 		}
 	}
-
+	return name
-	return NameFromString(name)
 }
 
 // NameFromString splits name into base file name and directory path.

api/layer/util_test.go

@@ -3,7 +3,6 @@ package layer
 import (
 	"net/http"
 	"strconv"
-	"strings"
 	"testing"
 	"time"
 
@@ -40,7 +39,7 @@ func newTestObject(oid *object.ID, bkt *BucketInfo, name string) *object.Object
 	return raw.Object()
 }
 
-func newTestInfo(oid *object.ID, bkt *BucketInfo, name, prefix string) *ObjectInfo {
+func newTestInfo(oid *object.ID, bkt *BucketInfo, name string, isDir bool) *ObjectInfo {
 	info := &ObjectInfo{
 		id:          oid,
 		Name:        name,
@@ -52,34 +51,16 @@ func newTestInfo(oid *object.ID, bkt *BucketInfo, name, prefix string) *ObjectIn
 		Headers:     make(map[string]string),
 	}
 
-	if prefix == rootSeparator {
+	if isDir {
-		return info
-	}
-
-	_, dirname := testNameFromObjectName(name)
-	if ln := len(prefix); ln > 0 && prefix[ln-1:] != PathSeparator {
-		prefix += PathSeparator
-	}
-
-	tail := strings.TrimPrefix(dirname, prefix)
-	if index := strings.Index(tail, PathSeparator); index >= 0 {
 		info.isDir = true
-
 		info.Size = 0
 		info.ContentType = ""
-		info.Name = tail[:index+1]
 		info.Headers = nil
 	}
 
 	return info
 }
 
-func testNameFromObjectName(name string) (string, string) {
-	ind := strings.LastIndex(name, PathSeparator)
-
-	return name[ind+1:], name[:ind+1]
-}
-
 func Test_objectInfoFromMeta(t *testing.T) {
 	uid := owner.NewID()
 	oid := object.NewID()
@@ -97,95 +78,77 @@ func Test_objectInfoFromMeta(t *testing.T) {
 		prefix    string
 		result    *ObjectInfo
 		object    *object.Object
-
+		delimiter string
-		infoName string
 	}{
 		{
-			name:     "test.jpg",
+			name:   "small.jpg",
-			prefix:   "",
+			result: newTestInfo(oid, bkt, "small.jpg", false),
-			infoName: "test.jpg",
+			object: newTestObject(oid, bkt, "small.jpg"),
-			result:   newTestInfo(oid, bkt, "test.jpg", ""),
+		},
-			object:   newTestObject(oid, bkt, "test.jpg"),
+		{
+			name:   "small.jpg not matched prefix",
+			prefix: "big",
+			result: nil,
+			object: newTestObject(oid, bkt, "small.jpg"),
+		},
+		{
+			name:      "small.jpg delimiter",
+			delimiter: "/",
+			result:    newTestInfo(oid, bkt, "small.jpg", false),
+			object:    newTestObject(oid, bkt, "small.jpg"),
 		},
-
 		{
 			name:   "test/small.jpg",
-			prefix:   "",
+			result: newTestInfo(oid, bkt, "test/small.jpg", false),
-			infoName: "test/",
-			result:   newTestInfo(oid, bkt, "test/small.jpg", ""),
 			object: newTestObject(oid, bkt, "test/small.jpg"),
 		},
-
 		{
-			name:     "test/small.jpg raw",
+			name:      "test/small.jpg with prefix and delimiter",
-			prefix:   rootSeparator,
+			prefix:    "test/",
-			infoName: "test/small.jpg",
+			delimiter: "/",
-			result:   newTestInfo(oid, bkt, "test/small.jpg", rootSeparator),
+			result:    newTestInfo(oid, bkt, "test/small.jpg", false),
 			object:    newTestObject(oid, bkt, "test/small.jpg"),
 		},
-
 		{
-			name:     "test/a/b/c/d/e/f/g/h/small.jpg",
+			name:   "a/b/small.jpg",
-			prefix:   "",
+			prefix: "a",
-			infoName: "test/",
+			result: newTestInfo(oid, bkt, "a/b/small.jpg", false),
-			result:   newTestInfo(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg", ""),
+			object: newTestObject(oid, bkt, "a/b/small.jpg"),
-			object:   newTestObject(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg"),
 		},
-
 		{
-			name:     "test/a/b/c/d/e/f/g/h/small.jpg",
+			name:      "a/b/small.jpg",
-			prefix:   "test",
+			prefix:    "a/",
-			infoName: "a/",
+			delimiter: "/",
-			result:   newTestInfo(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg", "test"),
+			result:    newTestInfo(oid, bkt, "a/b/", true),
-			object:   newTestObject(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg"),
+			object:    newTestObject(oid, bkt, "a/b/small.jpg"),
 		},
-
 		{
-			name:     "test/a/b/c/d/e/f/g/h/small.jpg",
+			name:      "a/b/c/small.jpg",
-			prefix:   "test/a",
+			prefix:    "a/",
-			infoName: "b/",
+			delimiter: "/",
-			result:   newTestInfo(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg", "test/a"),
+			result:    newTestInfo(oid, bkt, "a/b/", true),
-			object:   newTestObject(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg"),
+			object:    newTestObject(oid, bkt, "a/b/c/small.jpg"),
 		},
-
 		{
-			name:     "test/a/b/c/d/e/f/g/h/small.jpg",
+			name:      "a/b/c/small.jpg",
-			prefix:   "test/a/b",
+			prefix:    "a/b/c/s",
-			infoName: "c/",
+			delimiter: "/",
-			result:   newTestInfo(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg", "test/a/b"),
+			result:    newTestInfo(oid, bkt, "a/b/c/small.jpg", false),
-			object:   newTestObject(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg"),
+			object:    newTestObject(oid, bkt, "a/b/c/small.jpg"),
 		},
-
 		{
-			name:     "test/a/b/c/d/e/f/g/h/small.jpg with slash",
+			name:      "a/b/c/big.jpg",
-			prefix:   "test/a/b/",
+			prefix:    "a/b/",
-			infoName: "c/",
+			delimiter: "/",
-			result:   newTestInfo(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg", "test/a/b/"),
+			result:    newTestInfo(oid, bkt, "a/b/c/", true),
-			object:   newTestObject(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg"),
+			object:    newTestObject(oid, bkt, "a/b/c/big.jpg"),
-		},
-
-		{
-			name:     "test/a/b/c/d/e/f/g/h/small.jpg",
-			prefix:   "test/a/b/c",
-			infoName: "d/",
-			result:   newTestInfo(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg", "test/a/b/c"),
-			object:   newTestObject(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg"),
-		},
-
-		{
-			name:     "test/a/b/c/d/e/f/g/h/small.jpg",
-			prefix:   "test/a/b/c/d",
-			infoName: "e/",
-			result:   newTestInfo(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg", "test/a/b/c/d"),
-			object:   newTestObject(oid, bkt, "test/a/b/c/d/e/f/g/h/small.jpg"),
 		},
 	}
 
 	for _, tc := range cases {
-		t.Run(tc.name+"_"+tc.infoName, func(t *testing.T) {
+		t.Run(tc.name, func(t *testing.T) {
-			info := objectInfoFromMeta(bkt, tc.object, tc.prefix)
+			info := objectInfoFromMeta(bkt, tc.object, tc.prefix, tc.delimiter)
 			require.Equal(t, tc.result, info)
-			require.Equal(t, tc.infoName, info.Name)
 		})
 	}
 }