Merge pull request #172 from masterSplinter01/154-refactoring

Refactor ListObjectsV1 and V2

api/handler/head.go

@@ -32,9 +32,13 @@ func (h *handler) checkIsFolder(ctx context.Context, bucket, object string) *lay
 	}
 
 	_, dirname := layer.NameFromString(object)
-	params := &layer.ListObjectsParams{Bucket: bucket, Prefix: dirname, Delimiter: layer.PathSeparator}
+	params := &layer.ListObjectsParamsV1{
-
+		ListObjectsParamsCommon: layer.ListObjectsParamsCommon{
-	if list, err := h.obj.ListObjects(ctx, params); err == nil && len(list.Objects) > 0 {
+			Bucket:    bucket,
+			Prefix:    dirname,
+			Delimiter: layer.PathSeparator,
+		}}
+	if list, err := h.obj.ListObjectsV1(ctx, params); err == nil && len(list.Objects) > 0 {
 		return &layer.ObjectInfo{
 			Bucket: bucket,
 			Name:   object,

api/handler/list.go

@@ -3,26 +3,13 @@ package handler
 import (
 	"encoding/xml"
 	"net/http"
-	"strconv"
 	"time"
 
 	"github.com/nspcc-dev/neofs-api-go/pkg/owner"
 	"github.com/nspcc-dev/neofs-s3-gw/api"
-	"github.com/nspcc-dev/neofs-s3-gw/api/layer"
 	"go.uber.org/zap"
 )
 
-type listObjectsArgs struct {
-	Bucket     string
-	Delimiter  string
-	Encode     string
-	Marker     string
-	StartAfter string
-	MaxKeys    int
-	Prefix     string
-	APIVersion int
-}
-
 // VersioningConfiguration contains VersioningConfiguration XML representation.
 type VersioningConfiguration struct {
 	XMLName xml.Name `xml:"VersioningConfiguration"`
@@ -45,11 +32,7 @@ func (h *handler) registerAndSendError(w http.ResponseWriter, r *http.Request, e
 		zap.String("request_id", rid),
 		zap.Error(err))
 
-	api.WriteErrorResponse(r.Context(), w, api.Error{
+	api.WriteErrorResponse(r.Context(), w, err, r.URL)
-		Code:           api.GetAPIError(api.ErrBadRequest).Code,
-		Description:    err.Error(),
-		HTTPStatusCode: http.StatusBadRequest,
-	}, r.URL)
 }
 
 // ListBucketsHandler handles bucket listing requests.
@@ -107,207 +90,6 @@ func (h *handler) ListBucketsHandler(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
-func (h *handler) listObjects(w http.ResponseWriter, r *http.Request) (*listObjectsArgs, *layer.ListObjectsInfo, error) {
-	var (
-		err error
-		arg *listObjectsArgs
-		rid = api.GetRequestID(r.Context())
-	)
-
-	if arg, err = parseListObjectArgs(r); err != nil {
-		h.log.Error("something went wrong",
-			zap.String("request_id", rid),
-			zap.Error(err))
-
-		api.WriteErrorResponse(r.Context(), w, err, r.URL)
-		return nil, nil, err
-	}
-
-	marker := arg.Marker
-	if arg.APIVersion == 2 {
-		marker = arg.StartAfter
-	}
-
-	list, err := h.obj.ListObjects(r.Context(), &layer.ListObjectsParams{
-		Bucket:    arg.Bucket,
-		Prefix:    arg.Prefix,
-		MaxKeys:   arg.MaxKeys,
-		Delimiter: arg.Delimiter,
-		Marker:    marker,
-		Version:   arg.APIVersion,
-	})
-	if err != nil {
-		h.log.Error("something went wrong",
-			zap.String("request_id", rid),
-			zap.Error(err))
-
-		api.WriteErrorResponse(r.Context(), w, api.Error{
-			Code:           api.GetAPIError(api.ErrInternalError).Code,
-			Description:    err.Error(),
-			HTTPStatusCode: http.StatusInternalServerError,
-		}, r.URL)
-
-		return nil, nil, err
-	}
-
-	return arg, list, nil
-}
-
-// ListObjectsV1Handler handles objects listing requests for API version 1.
-func (h *handler) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {
-	var rid = api.GetRequestID(r.Context())
-	if arg, list, err := h.listObjects(w, r); err != nil {
-		// error already sent to client
-		return
-	} else if err := api.EncodeToResponse(w, encodeV1(arg, list)); err != nil {
-		h.log.Error("something went wrong",
-			zap.String("request_id", rid),
-			zap.Error(err))
-
-		api.WriteErrorResponse(r.Context(), w, api.Error{
-			Code:           api.GetAPIError(api.ErrInternalError).Code,
-			Description:    err.Error(),
-			HTTPStatusCode: http.StatusInternalServerError,
-		}, r.URL)
-	}
-}
-
-func encodeV1(arg *listObjectsArgs, list *layer.ListObjectsInfo) *ListObjectsResponse {
-	res := &ListObjectsResponse{
-		Name:         arg.Bucket,
-		EncodingType: arg.Encode,
-		Marker:       arg.Marker,
-		Prefix:       arg.Prefix,
-		MaxKeys:      arg.MaxKeys,
-		Delimiter:    arg.Delimiter,
-
-		IsTruncated: list.IsTruncated,
-		NextMarker:  list.NextMarker,
-	}
-
-	// fill common prefixes
-	for i := range list.Prefixes {
-		res.CommonPrefixes = append(res.CommonPrefixes, CommonPrefix{
-			Prefix: s3PathEncode(list.Prefixes[i], arg.Encode),
-		})
-	}
-
-	// fill contents
-	for _, obj := range list.Objects {
-		res.Contents = append(res.Contents, Object{
-			Key:          s3PathEncode(obj.Name, arg.Encode),
-			Size:         obj.Size,
-			LastModified: obj.Created.Format(time.RFC3339),
-
-			Owner: Owner{
-				ID:          obj.Owner.String(),
-				DisplayName: obj.Owner.String(),
-			},
-
-			ETag: obj.HashSum,
-			// StorageClass: "",
-		})
-	}
-
-	return res
-}
-
-// ListObjectsV2Handler handles objects listing requests for API version 2.
-func (h *handler) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) {
-	var rid = api.GetRequestID(r.Context())
-	if arg, list, err := h.listObjects(w, r); err != nil {
-		// error already sent to client
-		return
-	} else if err := api.EncodeToResponse(w, encodeV2(arg, list)); err != nil {
-		h.log.Error("something went wrong",
-			zap.String("request_id", rid),
-			zap.Error(err))
-
-		api.WriteErrorResponse(r.Context(), w, api.Error{
-			Code:           api.GetAPIError(api.ErrInternalError).Code,
-			Description:    err.Error(),
-			HTTPStatusCode: http.StatusInternalServerError,
-		}, r.URL)
-	}
-}
-
-func encodeV2(arg *listObjectsArgs, list *layer.ListObjectsInfo) *ListObjectsV2Response {
-	res := &ListObjectsV2Response{
-		Name:         arg.Bucket,
-		EncodingType: arg.Encode,
-		Prefix:       s3PathEncode(arg.Prefix, arg.Encode),
-		KeyCount:     len(list.Objects) + len(list.Prefixes),
-		MaxKeys:      arg.MaxKeys,
-		Delimiter:    s3PathEncode(arg.Delimiter, arg.Encode),
-		StartAfter:   s3PathEncode(arg.StartAfter, arg.Encode),
-
-		IsTruncated: list.IsTruncated,
-
-		ContinuationToken:     arg.Marker,
-		NextContinuationToken: list.NextContinuationToken,
-	}
-
-	// fill common prefixes
-	for i := range list.Prefixes {
-		res.CommonPrefixes = append(res.CommonPrefixes, CommonPrefix{
-			Prefix: s3PathEncode(list.Prefixes[i], arg.Encode),
-		})
-	}
-
-	// fill contents
-	for _, obj := range list.Objects {
-		res.Contents = append(res.Contents, Object{
-			Key:          s3PathEncode(obj.Name, arg.Encode),
-			Size:         obj.Size,
-			LastModified: obj.Created.Format(time.RFC3339),
-
-			Owner: Owner{
-				ID:          obj.Owner.String(),
-				DisplayName: obj.Owner.String(),
-			},
-
-			ETag: obj.HashSum,
-			// StorageClass: "",
-		})
-	}
-
-	return res
-}
-
-func parseListObjectArgs(r *http.Request) (*listObjectsArgs, error) {
-	var (
-		err error
-		res listObjectsArgs
-	)
-
-	if r.URL.Query().Get("max-keys") == "" {
-		res.MaxKeys = maxObjectList
-	} else if res.MaxKeys, err = strconv.Atoi(r.URL.Query().Get("max-keys")); err != nil || res.MaxKeys < 0 {
-		return nil, api.GetAPIError(api.ErrInvalidMaxKeys)
-	}
-
-	res.Prefix = r.URL.Query().Get("prefix")
-	res.Marker = r.URL.Query().Get("marker")
-	res.Delimiter = r.URL.Query().Get("delimiter")
-	res.Encode = r.URL.Query().Get("encoding-type")
-	res.StartAfter = r.URL.Query().Get("start-after")
-	apiVersionStr := r.URL.Query().Get("list-type")
-
-	res.APIVersion = 1
-	if len(apiVersionStr) != 0 {
-		if apiVersion, err := strconv.Atoi(apiVersionStr); err != nil || apiVersion != 2 {
-			return nil, api.GetAPIError(api.ErrIllegalVersioningConfigurationException)
-		}
-		res.APIVersion = 2
-	}
-
-	if info := api.GetReqInfo(r.Context()); info != nil {
-		res.Bucket = info.BucketName
-	}
-
-	return &res, nil
-}
-
 // GetBucketVersioningHandler implements bucket versioning getter handler.
 func (h *handler) GetBucketVersioningHandler(w http.ResponseWriter, r *http.Request) {
 	var (
@@ -351,92 +133,3 @@ func (h *handler) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Req
 		}, r.URL)
 	}
 }
-
-func (h *handler) ListBucketObjectVersionsHandler(w http.ResponseWriter, r *http.Request) {
-	p, err := parseListObjectVersionsRequest(r)
-	if err != nil {
-		h.registerAndSendError(w, r, err, "failed to parse request ")
-		return
-	}
-
-	info, err := h.obj.ListObjectVersions(r.Context(), p)
-	if err != nil {
-		h.registerAndSendError(w, r, err, "something went wrong")
-		return
-	}
-
-	response := encodeListObjectVersionsToResponse(info, p.Bucket)
-	if err := api.EncodeToResponse(w, response); err != nil {
-		h.registerAndSendError(w, r, err, "something went wrong")
-	}
-}
-
-func parseListObjectVersionsRequest(r *http.Request) (*layer.ListObjectVersionsParams, error) {
-	var (
-		err error
-		res layer.ListObjectVersionsParams
-	)
-
-	if r.URL.Query().Get("max-keys") == "" {
-		res.MaxKeys = maxObjectList
-	} else if res.MaxKeys, err = strconv.Atoi(r.URL.Query().Get("max-keys")); err != nil || res.MaxKeys < 0 {
-		return nil, api.GetAPIError(api.ErrInvalidMaxKeys)
-	}
-
-	res.Prefix = r.URL.Query().Get("prefix")
-	res.KeyMarker = r.URL.Query().Get("marker")
-	res.Delimiter = r.URL.Query().Get("delimiter")
-	res.Encode = r.URL.Query().Get("encoding-type")
-	res.VersionIDMarker = r.URL.Query().Get("version-id-marker")
-
-	if info := api.GetReqInfo(r.Context()); info != nil {
-		res.Bucket = info.BucketName
-	}
-
-	return &res, nil
-}
-
-func encodeListObjectVersionsToResponse(info *layer.ListObjectVersionsInfo, bucketName string) *ListObjectsVersionsResponse {
-	res := ListObjectsVersionsResponse{
-		Name:                bucketName,
-		IsTruncated:         info.IsTruncated,
-		KeyMarker:           info.KeyMarker,
-		NextKeyMarker:       info.NextKeyMarker,
-		NextVersionIDMarker: info.NextVersionIDMarker,
-		VersionIDMarker:     info.VersionIDMarker,
-	}
-
-	for _, prefix := range info.CommonPrefixes {
-		res.CommonPrefixes = append(res.CommonPrefixes, CommonPrefix{Prefix: *prefix})
-	}
-
-	for _, ver := range info.Version {
-		res.Version = append(res.Version, ObjectVersionResponse{
-			IsLatest:     ver.IsLatest,
-			Key:          ver.Object.Name,
-			LastModified: ver.Object.Created.Format(time.RFC3339),
-			Owner: Owner{
-				ID:          ver.Object.Owner.String(),
-				DisplayName: ver.Object.Owner.String(),
-			},
-			Size:      ver.Object.Size,
-			VersionID: ver.VersionID,
-			ETag:      ver.Object.HashSum,
-		})
-	}
-	// this loop is not starting till versioning is not implemented
-	for _, del := range info.DeleteMarker {
-		res.DeleteMarker = append(res.DeleteMarker, DeleteMarkerEntry{
-			IsLatest:     del.IsLatest,
-			Key:          del.Key,
-			LastModified: del.LastModified,
-			Owner: Owner{
-				ID:          del.Owner.String(),
-				DisplayName: del.Owner.String(),
-			},
-			VersionID: del.VersionID,
-		})
-	}
-
-	return &res
-}

api/handler/object_list.go

@@ -0,0 +1,269 @@
+package handler
+
+import (
+	"net/http"
+	"strconv"
+	"time"
+
+	"github.com/nspcc-dev/neofs-s3-gw/api"
+	"github.com/nspcc-dev/neofs-s3-gw/api/layer"
+)
+
+// ListObjectsV1Handler handles objects listing requests for API version 1.
+func (h *handler) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {
+	params, err := parseListObjectsArgsV1(r)
+	if err != nil {
+		h.registerAndSendError(w, r, err, "failed to parse arguments")
+		return
+	}
+
+	list, err := h.obj.ListObjectsV1(r.Context(), params)
+	if err != nil {
+		h.registerAndSendError(w, r, err, "something went wrong")
+		return
+	}
+
+	err = api.EncodeToResponse(w, encodeV1(params, list))
+	if err != nil {
+		h.registerAndSendError(w, r, err, "something went wrong")
+	}
+}
+
+func encodeV1(p *layer.ListObjectsParamsV1, list *layer.ListObjectsInfoV1) *ListObjectsV1Response {
+	res := &ListObjectsV1Response{
+		Name:         p.Bucket,
+		EncodingType: p.Encode,
+		Marker:       p.Marker,
+		Prefix:       p.Prefix,
+		MaxKeys:      p.MaxKeys,
+		Delimiter:    p.Delimiter,
+
+		IsTruncated: list.IsTruncated,
+		NextMarker:  list.NextMarker,
+	}
+
+	res.CommonPrefixes = fillPrefixes(list.Prefixes, p.Encode)
+
+	res.Contents = fillContents(list.Objects, p.Encode)
+
+	return res
+}
+
+// ListObjectsV2Handler handles objects listing requests for API version 2.
+func (h *handler) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) {
+	params, err := parseListObjectsArgsV2(r)
+	if err != nil {
+		h.registerAndSendError(w, r, err, "failed to parse arguments")
+		return
+	}
+
+	list, err := h.obj.ListObjectsV2(r.Context(), params)
+	if err != nil {
+		h.registerAndSendError(w, r, err, "something went wrong")
+		return
+	}
+
+	err = api.EncodeToResponse(w, encodeV2(params, list))
+	if err != nil {
+		h.registerAndSendError(w, r, err, "something went wrong")
+	}
+}
+
+func encodeV2(p *layer.ListObjectsParamsV2, list *layer.ListObjectsInfoV2) *ListObjectsV2Response {
+	res := &ListObjectsV2Response{
+		Name:         p.Bucket,
+		EncodingType: p.Encode,
+		Prefix:       s3PathEncode(p.Prefix, p.Encode),
+		KeyCount:     len(list.Objects) + len(list.Prefixes),
+		MaxKeys:      p.MaxKeys,
+		Delimiter:    s3PathEncode(p.Delimiter, p.Encode),
+		StartAfter:   s3PathEncode(p.StartAfter, p.Encode),
+
+		IsTruncated: list.IsTruncated,
+
+		ContinuationToken:     p.ContinuationToken,
+		NextContinuationToken: list.NextContinuationToken,
+	}
+
+	res.CommonPrefixes = fillPrefixes(list.Prefixes, p.Encode)
+
+	res.Contents = fillContents(list.Objects, p.Encode)
+
+	return res
+}
+
+func parseListObjectsArgsV1(r *http.Request) (*layer.ListObjectsParamsV1, error) {
+	var (
+		err error
+		res layer.ListObjectsParamsV1
+	)
+
+	common, err := parseListObjectArgs(r)
+	if err != nil {
+		return nil, err
+	}
+	res.ListObjectsParamsCommon = *common
+
+	res.Marker = r.URL.Query().Get("marker")
+
+	return &res, nil
+}
+
+func parseListObjectsArgsV2(r *http.Request) (*layer.ListObjectsParamsV2, error) {
+	var (
+		err error
+		res layer.ListObjectsParamsV2
+	)
+
+	common, err := parseListObjectArgs(r)
+	if err != nil {
+		return nil, err
+	}
+	res.ListObjectsParamsCommon = *common
+
+	res.ContinuationToken = r.URL.Query().Get("continuation-token")
+	res.StartAfter = r.URL.Query().Get("start-after")
+	return &res, nil
+}
+
+func parseListObjectArgs(r *http.Request) (*layer.ListObjectsParamsCommon, error) {
+	var (
+		err error
+		res layer.ListObjectsParamsCommon
+	)
+
+	if info := api.GetReqInfo(r.Context()); info != nil {
+		res.Bucket = info.BucketName
+	}
+
+	res.Delimiter = r.URL.Query().Get("delimiter")
+	res.Encode = r.URL.Query().Get("encoding-type")
+
+	if r.URL.Query().Get("max-keys") == "" {
+		res.MaxKeys = maxObjectList
+	} else if res.MaxKeys, err = strconv.Atoi(r.URL.Query().Get("max-keys")); err != nil || res.MaxKeys < 0 {
+		return nil, api.GetAPIError(api.ErrInvalidMaxKeys)
+	}
+
+	res.Prefix = r.URL.Query().Get("prefix")
+
+	return &res, nil
+}
+
+func fillPrefixes(src []string, encode string) []CommonPrefix {
+	var dst []CommonPrefix
+	for _, obj := range src {
+		dst = append(dst, CommonPrefix{
+			Prefix: s3PathEncode(obj, encode),
+		})
+	}
+	return dst
+}
+
+func fillContents(src []*layer.ObjectInfo, encode string) []Object {
+	var dst []Object
+	for _, obj := range src {
+		dst = append(dst, Object{
+			Key:          s3PathEncode(obj.Name, encode),
+			Size:         obj.Size,
+			LastModified: obj.Created.Format(time.RFC3339),
+
+			Owner: Owner{
+				ID:          obj.Owner.String(),
+				DisplayName: obj.Owner.String(),
+			},
+
+			ETag: obj.HashSum,
+		})
+	}
+	return dst
+}
+
+func (h *handler) ListBucketObjectVersionsHandler(w http.ResponseWriter, r *http.Request) {
+	p, err := parseListObjectVersionsRequest(r)
+	if err != nil {
+		h.registerAndSendError(w, r, err, "failed to parse request ")
+		return
+	}
+
+	info, err := h.obj.ListObjectVersions(r.Context(), p)
+	if err != nil {
+		h.registerAndSendError(w, r, err, "something went wrong")
+		return
+	}
+
+	response := encodeListObjectVersionsToResponse(info, p.Bucket)
+	if err := api.EncodeToResponse(w, response); err != nil {
+		h.registerAndSendError(w, r, err, "something went wrong")
+	}
+}
+
+func parseListObjectVersionsRequest(r *http.Request) (*layer.ListObjectVersionsParams, error) {
+	var (
+		err error
+		res layer.ListObjectVersionsParams
+	)
+
+	if r.URL.Query().Get("max-keys") == "" {
+		res.MaxKeys = maxObjectList
+	} else if res.MaxKeys, err = strconv.Atoi(r.URL.Query().Get("max-keys")); err != nil || res.MaxKeys <= 0 {
+		return nil, api.GetAPIError(api.ErrInvalidMaxKeys)
+	}
+
+	res.Prefix = r.URL.Query().Get("prefix")
+	res.KeyMarker = r.URL.Query().Get("marker")
+	res.Delimiter = r.URL.Query().Get("delimiter")
+	res.Encode = r.URL.Query().Get("encoding-type")
+	res.VersionIDMarker = r.URL.Query().Get("version-id-marker")
+
+	if info := api.GetReqInfo(r.Context()); info != nil {
+		res.Bucket = info.BucketName
+	}
+
+	return &res, nil
+}
+
+func encodeListObjectVersionsToResponse(info *layer.ListObjectVersionsInfo, bucketName string) *ListObjectsVersionsResponse {
+	res := ListObjectsVersionsResponse{
+		Name:                bucketName,
+		IsTruncated:         info.IsTruncated,
+		KeyMarker:           info.KeyMarker,
+		NextKeyMarker:       info.NextKeyMarker,
+		NextVersionIDMarker: info.NextVersionIDMarker,
+		VersionIDMarker:     info.VersionIDMarker,
+	}
+
+	for _, prefix := range info.CommonPrefixes {
+		res.CommonPrefixes = append(res.CommonPrefixes, CommonPrefix{Prefix: *prefix})
+	}
+
+	for _, ver := range info.Version {
+		res.Version = append(res.Version, ObjectVersionResponse{
+			IsLatest:     ver.IsLatest,
+			Key:          ver.Object.Name,
+			LastModified: ver.Object.Created.Format(time.RFC3339),
+			Owner: Owner{
+				ID:          ver.Object.Owner.String(),
+				DisplayName: ver.Object.Owner.String(),
+			},
+			Size:      ver.Object.Size,
+			VersionID: ver.VersionID,
+			ETag:      ver.Object.HashSum,
+		})
+	}
+	// this loop is not starting till versioning is not implemented
+	for _, del := range info.DeleteMarker {
+		res.DeleteMarker = append(res.DeleteMarker, DeleteMarkerEntry{
+			IsLatest:     del.IsLatest,
+			Key:          del.Key,
+			LastModified: del.LastModified,
+			Owner: Owner{
+				ID:          del.Owner.String(),
+				DisplayName: del.Owner.String(),
+			},
+			VersionID: del.VersionID,
+		})
+	}
+
+	return &res
+}

api/handler/response.go

@@ -14,35 +14,36 @@ type ListBucketsResponse struct {
 	} // Buckets are nested
 }
 
-// ListObjectsV2Response - format for list objects response.
+// ListObjectsV1Response -- format for ListObjectsV1 response.
+type ListObjectsV1Response struct {
+	XMLName        xml.Name       `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"`
+	CommonPrefixes []CommonPrefix `xml:"CommonPrefixes"`
+	Contents       []Object       `xml:"Contents"`
+	Delimiter      string         `xml:"Delimiter,omitempty"`
+	EncodingType   string         `xml:"EncodingType,omitempty"`
+	IsTruncated    bool           `xml:"IsTruncated"`
+	Marker         string         `xml:"Marker"`
+	MaxKeys        int            `xml:"MaxKeys"`
+	Name           string         `xml:"Name"`
+	NextMarker     string         `xml:"NextMarker,omitempty"`
+	Prefix         string         `xml:"Prefix"`
+}
+
+// ListObjectsV2Response -- format for ListObjectsV2 response.
 type ListObjectsV2Response struct {
-	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"`
+	XMLName               xml.Name       `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"`
-
+	CommonPrefixes        []CommonPrefix `xml:"CommonPrefixes"`
-	Name       string
+	Contents              []Object       `xml:"Contents"`
-	Prefix     string
+	ContinuationToken     string         `xml:"ContinuationToken,omitempty"`
-	StartAfter string `xml:"StartAfter,omitempty"`
+	Delimiter             string         `xml:"Delimiter,omitempty"`
-	// When response is truncated (the IsTruncated element value in the response
+	EncodingType          string         `xml:"EncodingType,omitempty"`
-	// is true), you can use the key name in this field as marker in the subsequent
+	IsTruncated           bool           `xml:"IsTruncated"`
-	// request to get next set of objects. Server lists objects in alphabetical
+	KeyCount              int            `xml:"KeyCount"`
-	// order Note: This element is returned only if you have delimiter request parameter
+	MaxKeys               int            `xml:"MaxKeys"`
-	// specified. If response does not include the NextMaker and it is truncated,
+	Name                  string         `xml:"Name"`
-	// you can use the value of the last Key in the response as the marker in the
+	NextContinuationToken string         `xml:"NextContinuationToken,omitempty"`
-	// subsequent request to get the next set of object keys.
+	Prefix                string         `xml:"Prefix"`
-	ContinuationToken     string `xml:"ContinuationToken,omitempty"`
+	StartAfter            string         `xml:"StartAfter,omitempty"`
-	NextContinuationToken string `xml:"NextContinuationToken,omitempty"`
-
-	KeyCount  int
-	MaxKeys   int
-	Delimiter string `xml:"Delimiter,omitempty"`
-	// A flag that indicates whether or not ListObjects returned all of the results
-	// that satisfied the search criteria.
-	IsTruncated bool
-
-	Contents       []Object
-	CommonPrefixes []CommonPrefix
-
-	// Encoding type used to encode object keys in the response.
-	EncodingType string `xml:"EncodingType,omitempty"`
 }
 
 // Bucket container for bucket metadata.
@@ -57,37 +58,7 @@ type Owner struct {
 	DisplayName string
 }
 
-// ListObjectsResponse - format for list objects response.
+// CommonPrefix container for prefix response in ListObjects's response.
-type ListObjectsResponse struct {
-	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"`
-
-	Name   string
-	Prefix string
-	Marker string
-
-	// When response is truncated (the IsTruncated element value in the response
-	// is true), you can use the key name in this field as marker in the subsequent
-	// request to get next set of objects. Server lists objects in alphabetical
-	// order Note: This element is returned only if you have delimiter request parameter
-	// specified. If response does not include the NextMaker and it is truncated,
-	// you can use the value of the last Key in the response as the marker in the
-	// subsequent request to get the next set of object keys.
-	NextMarker string `xml:"NextMarker,omitempty"`
-
-	MaxKeys   int
-	Delimiter string `xml:"Delimiter,omitempty"`
-	// A flag that indicates whether or not ListObjects returned all of the results
-	// that satisfied the search criteria.
-	IsTruncated bool
-
-	Contents       []Object
-	CommonPrefixes []CommonPrefix
-
-	// Encoding type used to encode object keys in the response.
-	EncodingType string `xml:"EncodingType,omitempty"`
-}
-
-// CommonPrefix container for prefix response in ListObjectsResponse.
 type CommonPrefix struct {
 	Prefix string
 }

api/layer/container.go

@@ -28,17 +28,6 @@ type (
 		Owner   *owner.ID
 		Created time.Time
 	}
-
-	// ListObjectsParams represents object listing request parameters.
-	ListObjectsParams struct {
-		Bucket    string
-		Prefix    string
-		Token     string
-		Delimiter string
-		MaxKeys   int
-		Marker    string
-		Version   int
-	}
 )
 
 func (n *layer) containerInfo(ctx context.Context, cid *cid.ID) (*BucketInfo, error) {

api/layer/layer.go

@@ -8,7 +8,6 @@ import (
 	"io"
 	"net/url"
 	"sort"
-	"strings"
 	"time"
 
 	"github.com/nspcc-dev/neofs-api-go/pkg/client"
@@ -114,7 +113,8 @@ type (
 
 		CopyObject(ctx context.Context, p *CopyObjectParams) (*ObjectInfo, error)
 
-		ListObjects(ctx context.Context, p *ListObjectsParams) (*ListObjectsInfo, error)
+		ListObjectsV1(ctx context.Context, p *ListObjectsParamsV1) (*ListObjectsInfoV1, error)
+		ListObjectsV2(ctx context.Context, p *ListObjectsParamsV2) (*ListObjectsInfoV2, error)
 		ListObjectVersions(ctx context.Context, p *ListObjectVersionsParams) (*ListObjectVersionsInfo, error)
 
 		DeleteObject(ctx context.Context, bucket, object string) error
@@ -134,8 +134,6 @@ var (
 )
 
 const (
-	// ETag (hex encoded md5sum) of empty string.
-	emptyETag                  = "d41d8cd98f00b204e9800998ecf8427e"
 	unversionedObjectVersionID = "null"
 )
 
@@ -212,127 +210,6 @@ func (n *layer) ListBuckets(ctx context.Context) ([]*BucketInfo, error) {
 	return n.containerList(ctx)
 }
 
-// ListObjects returns objects from the container. It ignores tombstones and
-// storage groups.
-//                          ctx, bucket, prefix, continuationToken, delimiter, maxKeys
-func (n *layer) ListObjects(ctx context.Context, p *ListObjectsParams) (*ListObjectsInfo, error) {
-	// todo: make pagination when search response will be gRPC stream,
-	//       pagination must be implemented with cache, because search results
-	//       may be different between search calls
-	var (
-		err       error
-		bkt       *BucketInfo
-		ids       []*object.ID
-		result    ListObjectsInfo
-		uniqNames = make(map[string]bool)
-	)
-
-	if p.MaxKeys == 0 {
-		return &result, nil
-	}
-
-	if bkt, err = n.GetBucketInfo(ctx, p.Bucket); err != nil {
-		return nil, err
-	} else if ids, err = n.objectSearch(ctx, &findParams{cid: bkt.CID}); err != nil {
-		return nil, err
-	}
-
-	ln := len(ids)
-	// todo: check what happens if there is more than maxKeys objects
-	if ln > p.MaxKeys {
-		ln = p.MaxKeys
-	}
-
-	mostRecentModified := time.Time{}
-	needDirectoryAsKey := p.Version == 2 && len(p.Prefix) > 0 && len(p.Delimiter) > 0 && strings.HasSuffix(p.Prefix, p.Delimiter)
-	result.Objects = make([]*ObjectInfo, 0, ln)
-
-	for _, id := range ids {
-		addr := object.NewAddress()
-		addr.SetObjectID(id)
-		addr.SetContainerID(bkt.CID)
-
-		meta, err := n.objectHead(ctx, addr)
-		if err != nil {
-			n.log.Warn("could not fetch object meta", zap.Error(err))
-			continue
-		}
-
-		// // ignore tombstone objects
-		// _, hdr := meta.LastHeader(object.HeaderType(object.TombstoneHdr))
-		// if hdr != nil {
-		// 	continue
-		// }
-
-		// ignore storage group objects
-		// _, hdr = meta.LastHeader(object.HeaderType(object.StorageGroupHdr))
-		// if hdr != nil {
-		// 	continue
-		// }
-
-		// dirs don't exist in neofs, gateway stores full path to the file
-		// in object header, e.g. `filename`:`/this/is/path/file.txt`
-
-		// prefix argument contains full dir path from the root, e.g. `/this/is/`
-
-		// to emulate dirs we take dirs in path, compare it with prefix
-		// and look for entities after prefix. If entity does not have any
-		// sub-entities, then it is a file, else directory.
-
-		if oi := objectInfoFromMeta(bkt, meta, p.Prefix, p.Delimiter); oi != nil {
-			if needDirectoryAsKey && oi.Created.After(mostRecentModified) {
-				mostRecentModified = oi.Created
-			}
-			// use only unique dir names
-			if _, ok := uniqNames[oi.Name]; ok {
-				continue
-			}
-			if len(p.Marker) > 0 && oi.Name <= p.Marker {
-				continue
-			}
-
-			uniqNames[oi.Name] = oi.isDir
-
-			result.Objects = append(result.Objects, oi)
-		}
-	}
-
-	sort.Slice(result.Objects, func(i, j int) bool {
-		return result.Objects[i].Name < result.Objects[j].Name
-	})
-
-	if len(result.Objects) > p.MaxKeys {
-		result.IsTruncated = true
-		result.Objects = result.Objects[:p.MaxKeys]
-		result.NextMarker = result.Objects[len(result.Objects)-1].Name
-	}
-
-	fillPrefixes(&result, uniqNames)
-	if needDirectoryAsKey {
-		res := []*ObjectInfo{{
-			Name:    p.Prefix,
-			Created: mostRecentModified,
-			HashSum: emptyETag,
-		}}
-		result.Objects = append(res, result.Objects...)
-	}
-
-	return &result, nil
-}
-
-func fillPrefixes(result *ListObjectsInfo, directories map[string]bool) {
-	index := 0
-	for range result.Objects {
-		name := result.Objects[index].Name
-		if isDir := directories[name]; isDir {
-			result.Objects = append(result.Objects[:index], result.Objects[index+1:]...)
-			result.Prefixes = append(result.Prefixes, name)
-		} else {
-			index++
-		}
-	}
-}
-
 // GetObject from storage.
 func (n *layer) GetObject(ctx context.Context, p *GetObjectParams) error {
 	var (

api/layer/layer_test.go

@@ -1,44 +0,0 @@
-package layer
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/require"
-)
-
-func TestFillingPrefixes(t *testing.T) {
-	cases := []struct {
-		name             string
-		list             *ListObjectsInfo
-		directories      map[string]bool
-		expectedPrefixes []string
-		expectedObjects  []*ObjectInfo
-	}{
-		{
-			name: "3 dirs",
-			list: &ListObjectsInfo{
-				Objects: []*ObjectInfo{{Name: "dir/"}, {Name: "dir2/"}, {Name: "dir3/"}},
-			},
-			directories:      map[string]bool{"dir/": true, "dir2/": true, "dir3/": true},
-			expectedPrefixes: []string{"dir/", "dir2/", "dir3/"},
-			expectedObjects:  []*ObjectInfo{},
-		},
-		{
-			name: "1 obj, 3 dirs",
-			list: &ListObjectsInfo{
-				Objects: []*ObjectInfo{{Name: "dir/"}, {Name: "dir2/"}, {Name: "dir3/"}, {Name: "obj"}},
-			},
-			directories:      map[string]bool{"dir/": true, "dir2/": true, "dir3/": true},
-			expectedPrefixes: []string{"dir/", "dir2/", "dir3/"},
-			expectedObjects:  []*ObjectInfo{{Name: "obj"}},
-		},
-	}
-
-	for _, tc := range cases {
-		t.Run(tc.name, func(t *testing.T) {
-			fillPrefixes(tc.list, tc.directories)
-			require.Equal(t, tc.expectedPrefixes, tc.list.Prefixes)
-			require.Equal(t, tc.expectedObjects, tc.list.Objects)
-		})
-	}
-}

api/layer/object.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"io"
 	"net/url"
+	"sort"
 	"strconv"
 	"time"
 
@@ -12,6 +13,7 @@ import (
 	cid "github.com/nspcc-dev/neofs-api-go/pkg/container/id"
 	"github.com/nspcc-dev/neofs-api-go/pkg/object"
 	"github.com/nspcc-dev/neofs-s3-gw/api"
+	"go.uber.org/zap"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 )
@@ -30,6 +32,35 @@ type (
 		length  int64
 		address *object.Address
 	}
+
+	// ListObjectsParamsCommon contains common parameters for ListObjectsV1 and ListObjectsV2.
+	ListObjectsParamsCommon struct {
+		Bucket    string
+		Delimiter string
+		Encode    string
+		MaxKeys   int
+		Prefix    string
+	}
+
+	// ListObjectsParamsV1 contains params for ListObjectsV1.
+	ListObjectsParamsV1 struct {
+		ListObjectsParamsCommon
+		Marker string
+	}
+
+	// ListObjectsParamsV2 contains params for ListObjectsV2.
+	ListObjectsParamsV2 struct {
+		ListObjectsParamsCommon
+		ContinuationToken string
+		StartAfter        string
+	}
+
+	allObjectParams struct {
+		Bucket     string
+		Delimiter  string
+		Prefix     string
+		StartAfter string
+	}
 )
 
 // objectSearch returns all available objects by search params.
@@ -170,3 +201,135 @@ func (n *layer) objectDelete(ctx context.Context, address *object.Address) error
 	dop.WithAddress(address)
 	return n.pool.DeleteObject(ctx, dop, n.BearerOpt(ctx))
 }
+
+// ListObjectsV1 returns objects in a bucket for requests of Version 1.
+func (n *layer) ListObjectsV1(ctx context.Context, p *ListObjectsParamsV1) (*ListObjectsInfoV1, error) {
+	var (
+		err    error
+		result ListObjectsInfoV1
+	)
+
+	if p.MaxKeys == 0 {
+		return &result, nil
+	}
+
+	allObjects, err := n.listSortedAllObjects(ctx, allObjectParams{
+		Bucket:     p.Bucket,
+		Prefix:     p.Prefix,
+		Delimiter:  p.Delimiter,
+		StartAfter: p.Marker,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	if len(allObjects) > p.MaxKeys {
+		result.IsTruncated = true
+
+		nextObject := allObjects[p.MaxKeys-1]
+		result.NextMarker = nextObject.Name
+
+		allObjects = allObjects[:p.MaxKeys]
+	}
+
+	for _, ov := range allObjects {
+		if ov.isDir {
+			result.Prefixes = append(result.Prefixes, ov.Name)
+		} else {
+			result.Objects = append(result.Objects, ov)
+		}
+	}
+
+	return &result, nil
+}
+
+// ListObjectsV2 returns objects in a bucket for requests of Version 2.
+func (n *layer) ListObjectsV2(ctx context.Context, p *ListObjectsParamsV2) (*ListObjectsInfoV2, error) {
+	var (
+		err        error
+		result     ListObjectsInfoV2
+		allObjects []*ObjectInfo
+	)
+
+	if p.MaxKeys == 0 {
+		return &result, nil
+	}
+
+	if p.ContinuationToken != "" {
+		// find cache with continuation token
+	} else {
+		allObjects, err = n.listSortedAllObjects(ctx, allObjectParams{
+			Bucket:     p.Bucket,
+			Prefix:     p.Prefix,
+			Delimiter:  p.Delimiter,
+			StartAfter: p.StartAfter,
+		})
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if len(allObjects) > p.MaxKeys {
+		result.IsTruncated = true
+
+		allObjects = allObjects[:p.MaxKeys]
+		// add  creating of cache here
+	}
+
+	for _, ov := range allObjects {
+		if ov.isDir {
+			result.Prefixes = append(result.Prefixes, ov.Name)
+		} else {
+			result.Objects = append(result.Objects, ov)
+		}
+	}
+	return &result, nil
+}
+
+func (n *layer) listSortedAllObjects(ctx context.Context, p allObjectParams) ([]*ObjectInfo, error) {
+	var (
+		err       error
+		bkt       *BucketInfo
+		ids       []*object.ID
+		uniqNames = make(map[string]bool)
+	)
+
+	if bkt, err = n.GetBucketInfo(ctx, p.Bucket); err != nil {
+		return nil, err
+	} else if ids, err = n.objectSearch(ctx, &findParams{cid: bkt.CID}); err != nil {
+		return nil, err
+	}
+
+	objects := make([]*ObjectInfo, 0, len(ids))
+
+	for _, id := range ids {
+		addr := object.NewAddress()
+		addr.SetObjectID(id)
+		addr.SetContainerID(bkt.CID)
+
+		meta, err := n.objectHead(ctx, addr)
+		if err != nil {
+			n.log.Warn("could not fetch object meta", zap.Error(err))
+			continue
+		}
+		if oi := objectInfoFromMeta(bkt, meta, p.Prefix, p.Delimiter); oi != nil {
+			// use only unique dir names
+			if _, ok := uniqNames[oi.Name]; ok {
+				continue
+			}
+			if len(p.StartAfter) > 0 && oi.Name <= p.StartAfter {
+				continue
+			}
+
+			uniqNames[oi.Name] = oi.isDir
+
+			objects = append(objects, oi)
+		}
+	}
+
+	sort.Slice(objects, func(i, j int) bool {
+		return objects[i].Name < objects[j].Name
+	})
+
+	return objects, nil
+}

api/layer/util.go

@@ -26,32 +26,23 @@ type (
 		Headers     map[string]string
 	}
 
-	// ListObjectsInfo - container for list objects.
+	// ListObjectsInfo contains common fields of data for ListObjectsV1 and ListObjectsV2.
 	ListObjectsInfo struct {
-		// Indicates whether the returned list objects response is truncated. A
+		Prefixes    []string
-		// value of true indicates that the list was truncated. The list can be truncated
+		Objects     []*ObjectInfo
-		// if the number of objects exceeds the limit allowed or specified
-		// by max keys.
 		IsTruncated bool
+	}
 
-		// When response is truncated (the IsTruncated element value in the response
+	// ListObjectsInfoV1 holds data which ListObjectsV1 returns.
-		// is true), you can use the key name in this field as marker in the subsequent
+	ListObjectsInfoV1 struct {
-		// request to get next set of objects.
+		ListObjectsInfo
-		//
-		// NOTE: This element is returned only if you have delimiter request parameter
-		// specified.
-		ContinuationToken     string
-		NextContinuationToken string
-
-		// When response is truncated (the IsTruncated element value in the response is true),
-		// you can use the key name in this field as marker in the subsequent request to get next set of objects.
 		NextMarker string
+	}
 
-		// List of objects info for this request.
+	// ListObjectsInfoV2 holds data which ListObjectsV2 returns.
-		Objects []*ObjectInfo
+	ListObjectsInfoV2 struct {
-
+		ListObjectsInfo
-		// List of prefixes for this request.
+		NextContinuationToken string
-		Prefixes []string
 	}
 
 	// ObjectVersionInfo stores info about objects versions.