TrueCloudLab/frostfs-s3-gw
17
3
Fork 19
Code Issues 28 Pull requests 7 Projects Releases 18 Packages Activity Actions

[#154] api: refactor ListObjectsV1 and V2

Browse source 
Separate ListObject for different versions.
Remove useless grouping of keys on V2.

Signed-off-by: Angira Kekteeva <kira@nspcc.ru>
This commit is contained in:
Angira Kekteeva 2021-07-18 16:40:19 +03:00
parent 308994211b
commit ee84062154
9 changed files with 493 additions and 571 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
api/handler/head.go
View file

@ -32,9 +32,13 @@ func (h *handler) checkIsFolder(ctx context.Context, bucket, object string) *lay
}
_, dirname := layer.NameFromString(object)
params := &layer.ListObjectsParams{Bucket: bucket, Prefix: dirname, Delimiter: layer.PathSeparator}
if list, err := h.obj.ListObjects(ctx, params); err == nil && len(list.Objects) > 0 {
params := &layer.ListObjectsParamsV1{
ListObjectsParamsCommon: layer.ListObjectsParamsCommon{
Bucket: bucket,
Prefix: dirname,
Delimiter: layer.PathSeparator,
}}
if list, err := h.obj.ListObjectsV1(ctx, params); err == nil && len(list.Objects) > 0 {
return &layer.ObjectInfo{
Bucket: bucket,
Name: object,

309
api/handler/list.go
View file

@ -3,26 +3,13 @@ package handler
import (
"encoding/xml"
"net/http"
"strconv"
"time"
"github.com/nspcc-dev/neofs-api-go/pkg/owner"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
"go.uber.org/zap"
)
type listObjectsArgs struct {
Bucket string
Delimiter string
Encode string
Marker string
StartAfter string
MaxKeys int
Prefix string
APIVersion int
}
// VersioningConfiguration contains VersioningConfiguration XML representation.
type VersioningConfiguration struct {
XMLName xml.Name `xml:"VersioningConfiguration"`
@ -45,11 +32,7 @@ func (h *handler) registerAndSendError(w http.ResponseWriter, r *http.Request, e
zap.String("request_id", rid),
zap.Error(err))
api.WriteErrorResponse(r.Context(), w, api.Error{
Code: api.GetAPIError(api.ErrBadRequest).Code,
Description: err.Error(),
HTTPStatusCode: http.StatusBadRequest,
}, r.URL)
api.WriteErrorResponse(r.Context(), w, err, r.URL)
}
// ListBucketsHandler handles bucket listing requests.
@ -107,207 +90,6 @@ func (h *handler) ListBucketsHandler(w http.ResponseWriter, r *http.Request) {
}
}
func (h *handler) listObjects(w http.ResponseWriter, r *http.Request) (*listObjectsArgs, *layer.ListObjectsInfo, error) {
var (
err error
arg *listObjectsArgs
rid = api.GetRequestID(r.Context())
)
if arg, err = parseListObjectArgs(r); err != nil {
h.log.Error("something went wrong",
zap.String("request_id", rid),
zap.Error(err))
api.WriteErrorResponse(r.Context(), w, err, r.URL)
return nil, nil, err
}
marker := arg.Marker
if arg.APIVersion == 2 {
marker = arg.StartAfter
}
list, err := h.obj.ListObjects(r.Context(), &layer.ListObjectsParams{
Bucket: arg.Bucket,
Prefix: arg.Prefix,
MaxKeys: arg.MaxKeys,
Delimiter: arg.Delimiter,
Marker: marker,
Version: arg.APIVersion,
})
if err != nil {
h.log.Error("something went wrong",
zap.String("request_id", rid),
zap.Error(err))
api.WriteErrorResponse(r.Context(), w, api.Error{
Code: api.GetAPIError(api.ErrInternalError).Code,
Description: err.Error(),
HTTPStatusCode: http.StatusInternalServerError,
}, r.URL)
return nil, nil, err
}
return arg, list, nil
}
// ListObjectsV1Handler handles objects listing requests for API version 1.
func (h *handler) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {
var rid = api.GetRequestID(r.Context())
if arg, list, err := h.listObjects(w, r); err != nil {
// error already sent to client
return
} else if err := api.EncodeToResponse(w, encodeV1(arg, list)); err != nil {
h.log.Error("something went wrong",
zap.String("request_id", rid),
zap.Error(err))
api.WriteErrorResponse(r.Context(), w, api.Error{
Code: api.GetAPIError(api.ErrInternalError).Code,
Description: err.Error(),
HTTPStatusCode: http.StatusInternalServerError,
}, r.URL)
}
}
func encodeV1(arg *listObjectsArgs, list *layer.ListObjectsInfo) *ListObjectsResponse {
res := &ListObjectsResponse{
Name: arg.Bucket,
EncodingType: arg.Encode,
Marker: arg.Marker,
Prefix: arg.Prefix,
MaxKeys: arg.MaxKeys,
Delimiter: arg.Delimiter,
IsTruncated: list.IsTruncated,
NextMarker: list.NextMarker,
}
// fill common prefixes
for i := range list.Prefixes {
res.CommonPrefixes = append(res.CommonPrefixes, CommonPrefix{
Prefix: s3PathEncode(list.Prefixes[i], arg.Encode),
})
}
// fill contents
for _, obj := range list.Objects {
res.Contents = append(res.Contents, Object{
Key: s3PathEncode(obj.Name, arg.Encode),
Size: obj.Size,
LastModified: obj.Created.Format(time.RFC3339),
Owner: Owner{
ID: obj.Owner.String(),
DisplayName: obj.Owner.String(),
},
ETag: obj.HashSum,
// StorageClass: "",
})
}
return res
}
// ListObjectsV2Handler handles objects listing requests for API version 2.
func (h *handler) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) {
var rid = api.GetRequestID(r.Context())
if arg, list, err := h.listObjects(w, r); err != nil {
// error already sent to client
return
} else if err := api.EncodeToResponse(w, encodeV2(arg, list)); err != nil {
h.log.Error("something went wrong",
zap.String("request_id", rid),
zap.Error(err))
api.WriteErrorResponse(r.Context(), w, api.Error{
Code: api.GetAPIError(api.ErrInternalError).Code,
Description: err.Error(),
HTTPStatusCode: http.StatusInternalServerError,
}, r.URL)
}
}
func encodeV2(arg *listObjectsArgs, list *layer.ListObjectsInfo) *ListObjectsV2Response {
res := &ListObjectsV2Response{
Name: arg.Bucket,
EncodingType: arg.Encode,
Prefix: s3PathEncode(arg.Prefix, arg.Encode),
KeyCount: len(list.Objects) + len(list.Prefixes),
MaxKeys: arg.MaxKeys,
Delimiter: s3PathEncode(arg.Delimiter, arg.Encode),
StartAfter: s3PathEncode(arg.StartAfter, arg.Encode),
IsTruncated: list.IsTruncated,
ContinuationToken: arg.Marker,
NextContinuationToken: list.NextContinuationToken,
}
// fill common prefixes
for i := range list.Prefixes {
res.CommonPrefixes = append(res.CommonPrefixes, CommonPrefix{
Prefix: s3PathEncode(list.Prefixes[i], arg.Encode),
})
}
// fill contents
for _, obj := range list.Objects {
res.Contents = append(res.Contents, Object{
Key: s3PathEncode(obj.Name, arg.Encode),
Size: obj.Size,
LastModified: obj.Created.Format(time.RFC3339),
Owner: Owner{
ID: obj.Owner.String(),
DisplayName: obj.Owner.String(),
},
ETag: obj.HashSum,
// StorageClass: "",
})
}
return res
}
func parseListObjectArgs(r *http.Request) (*listObjectsArgs, error) {
var (
err error
res listObjectsArgs
)
if r.URL.Query().Get("max-keys") == "" {
res.MaxKeys = maxObjectList
} else if res.MaxKeys, err = strconv.Atoi(r.URL.Query().Get("max-keys")); err != nil || res.MaxKeys < 0 {
return nil, api.GetAPIError(api.ErrInvalidMaxKeys)
}
res.Prefix = r.URL.Query().Get("prefix")
res.Marker = r.URL.Query().Get("marker")
res.Delimiter = r.URL.Query().Get("delimiter")
res.Encode = r.URL.Query().Get("encoding-type")
res.StartAfter = r.URL.Query().Get("start-after")
apiVersionStr := r.URL.Query().Get("list-type")
res.APIVersion = 1
if len(apiVersionStr) != 0 {
if apiVersion, err := strconv.Atoi(apiVersionStr); err != nil || apiVersion != 2 {
return nil, api.GetAPIError(api.ErrIllegalVersioningConfigurationException)
}
res.APIVersion = 2
}
if info := api.GetReqInfo(r.Context()); info != nil {
res.Bucket = info.BucketName
}
return &res, nil
}
// GetBucketVersioningHandler implements bucket versioning getter handler.
func (h *handler) GetBucketVersioningHandler(w http.ResponseWriter, r *http.Request) {
var (
@ -351,92 +133,3 @@ func (h *handler) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Req
}, r.URL)
}
}
func (h *handler) ListBucketObjectVersionsHandler(w http.ResponseWriter, r *http.Request) {
p, err := parseListObjectVersionsRequest(r)
if err != nil {
h.registerAndSendError(w, r, err, "failed to parse request ")
return
}
info, err := h.obj.ListObjectVersions(r.Context(), p)
if err != nil {
h.registerAndSendError(w, r, err, "something went wrong")
return
}
response := encodeListObjectVersionsToResponse(info, p.Bucket)
if err := api.EncodeToResponse(w, response); err != nil {
h.registerAndSendError(w, r, err, "something went wrong")
}
}
func parseListObjectVersionsRequest(r *http.Request) (*layer.ListObjectVersionsParams, error) {
var (
err error
res layer.ListObjectVersionsParams
)
if r.URL.Query().Get("max-keys") == "" {
res.MaxKeys = maxObjectList
} else if res.MaxKeys, err = strconv.Atoi(r.URL.Query().Get("max-keys")); err != nil || res.MaxKeys < 0 {
return nil, api.GetAPIError(api.ErrInvalidMaxKeys)
}
res.Prefix = r.URL.Query().Get("prefix")
res.KeyMarker = r.URL.Query().Get("marker")
res.Delimiter = r.URL.Query().Get("delimiter")
res.Encode = r.URL.Query().Get("encoding-type")
res.VersionIDMarker = r.URL.Query().Get("version-id-marker")
if info := api.GetReqInfo(r.Context()); info != nil {
res.Bucket = info.BucketName
}
return &res, nil
}
func encodeListObjectVersionsToResponse(info *layer.ListObjectVersionsInfo, bucketName string) *ListObjectsVersionsResponse {
res := ListObjectsVersionsResponse{
Name: bucketName,
IsTruncated: info.IsTruncated,
KeyMarker: info.KeyMarker,
NextKeyMarker: info.NextKeyMarker,
NextVersionIDMarker: info.NextVersionIDMarker,
VersionIDMarker: info.VersionIDMarker,
}
for _, prefix := range info.CommonPrefixes {
res.CommonPrefixes = append(res.CommonPrefixes, CommonPrefix{Prefix: *prefix})
}
for _, ver := range info.Version {
res.Version = append(res.Version, ObjectVersionResponse{
IsLatest: ver.IsLatest,
Key: ver.Object.Name,
LastModified: ver.Object.Created.Format(time.RFC3339),
Owner: Owner{
ID: ver.Object.Owner.String(),
DisplayName: ver.Object.Owner.String(),
},
Size: ver.Object.Size,
VersionID: ver.VersionID,
ETag: ver.Object.HashSum,
})
}
// this loop is not starting till versioning is not implemented
for _, del := range info.DeleteMarker {
res.DeleteMarker = append(res.DeleteMarker, DeleteMarkerEntry{
IsLatest: del.IsLatest,
Key: del.Key,
LastModified: del.LastModified,
Owner: Owner{
ID: del.Owner.String(),
DisplayName: del.Owner.String(),
},
VersionID: del.VersionID,
})
}
return &res
}

278
api/handler/object_list.go Normal file
View file

@ -0,0 +1,278 @@
package handler
import (
"net/http"
"strconv"
"time"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
)
// ListObjectsV1Handler handles objects listing requests for API version 1.
func (h *handler) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {
params, err := parseListObjectsArgsV1(r)
if err != nil {
h.registerAndSendError(w, r, err, "failed to parse arguments")
return
}
list, err := h.obj.ListObjectsV1(r.Context(), params)
if err != nil {
h.registerAndSendError(w, r, err, "something went wrong")
return
}
err = api.EncodeToResponse(w, encodeV1(params, list))
if err != nil {
h.registerAndSendError(w, r, err, "something went wrong")
}
}
func encodeV1(p *layer.ListObjectsParamsV1, list *layer.ListObjectsInfoV1) *ListObjectsV1Response {
res := &ListObjectsV1Response{
Name: p.Bucket,
EncodingType: p.Encode,
Marker: p.Marker,
Prefix: p.Prefix,
MaxKeys: p.MaxKeys,
Delimiter: p.Delimiter,
IsTruncated: list.IsTruncated,
NextMarker: list.NextMarker,
}
// fill common prefixes
for i := range list.Prefixes {
res.CommonPrefixes = append(res.CommonPrefixes, CommonPrefix{
Prefix: s3PathEncode(list.Prefixes[i], p.Encode),
})
}
// fill contents
for _, obj := range list.Objects {
res.Contents = append(res.Contents, Object{
Key: s3PathEncode(obj.Name, p.Encode),
Size: obj.Size,
LastModified: obj.Created.Format(time.RFC3339),
Owner: Owner{
ID: obj.Owner.String(),
DisplayName: obj.Owner.String(),
},
ETag: obj.HashSum,
})
}
return res
}
// ListObjectsV2Handler handles objects listing requests for API version 2.
func (h *handler) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) {
params, err := parseListObjectsArgsV2(r)
if err != nil {
h.registerAndSendError(w, r, err, "failed to parse arguments")
return
}
list, err := h.obj.ListObjectsV2(r.Context(), params)
if err != nil {
h.registerAndSendError(w, r, err, "something went wrong")
return
}
err = api.EncodeToResponse(w, encodeV2(params, list))
if err != nil {
h.registerAndSendError(w, r, err, "something went wrong")
}
}
func encodeV2(p *layer.ListObjectsParamsV2, list *layer.ListObjectsInfoV2) *ListObjectsV2Response {
res := &ListObjectsV2Response{
Name: p.Bucket,
EncodingType: p.Encode,
Prefix: s3PathEncode(p.Prefix, p.Encode),
KeyCount: len(list.Objects) + len(list.Prefixes),
MaxKeys: p.MaxKeys,
Delimiter: s3PathEncode(p.Delimiter, p.Encode),
StartAfter: s3PathEncode(p.StartAfter, p.Encode),
IsTruncated: list.IsTruncated,
ContinuationToken: p.ContinuationToken,
NextContinuationToken: list.NextContinuationToken,
}
// fill common prefixes
for i := range list.Prefixes {
res.CommonPrefixes = append(res.CommonPrefixes, CommonPrefix{
Prefix: s3PathEncode(list.Prefixes[i], p.Encode),
})
}
// fill contents
for _, obj := range list.Objects {
res.Contents = append(res.Contents, Object{
Key: s3PathEncode(obj.Name, p.Encode),
Size: obj.Size,
LastModified: obj.Created.Format(time.RFC3339),
Owner: Owner{
ID: obj.Owner.String(),
DisplayName: obj.Owner.String(),
},
ETag: obj.HashSum,
})
}
return res
}
func parseListObjectsArgsV1(r *http.Request) (*layer.ListObjectsParamsV1, error) {
var (
err error
res layer.ListObjectsParamsV1
)
common, err := parseListObjectArgs(r)
if err != nil {
return nil, err
}
res.ListObjectsParamsCommon = *common
res.Marker = r.URL.Query().Get("marker")
return &res, nil
}
func parseListObjectsArgsV2(r *http.Request) (*layer.ListObjectsParamsV2, error) {
var (
err error
res layer.ListObjectsParamsV2
)
common, err := parseListObjectArgs(r)
if err != nil {
return nil, err
}
res.ListObjectsParamsCommon = *common
res.ContinuationToken = r.URL.Query().Get("continuation-token")
res.StartAfter = r.URL.Query().Get("start-after")
return &res, nil
}
func parseListObjectArgs(r *http.Request) (*layer.ListObjectsParamsCommon, error) {
var (
err error
res layer.ListObjectsParamsCommon
)
if info := api.GetReqInfo(r.Context()); info != nil {
res.Bucket = info.BucketName
}
res.Delimiter = r.URL.Query().Get("delimiter")
res.Encode = r.URL.Query().Get("encoding-type")
if r.URL.Query().Get("max-keys") == "" {
res.MaxKeys = maxObjectList
} else if res.MaxKeys, err = strconv.Atoi(r.URL.Query().Get("max-keys")); err != nil || res.MaxKeys < 0 {
return nil, api.GetAPIError(api.ErrInvalidMaxKeys)
}
res.Prefix = r.URL.Query().Get("prefix")
return &res, nil
}
func (h *handler) ListBucketObjectVersionsHandler(w http.ResponseWriter, r *http.Request) {
p, err := parseListObjectVersionsRequest(r)
if err != nil {
h.registerAndSendError(w, r, err, "failed to parse request ")
return
}
info, err := h.obj.ListObjectVersions(r.Context(), p)
if err != nil {
h.registerAndSendError(w, r, err, "something went wrong")
return
}
response := encodeListObjectVersionsToResponse(info, p.Bucket)
if err := api.EncodeToResponse(w, response); err != nil {
h.registerAndSendError(w, r, err, "something went wrong")
}
}
func parseListObjectVersionsRequest(r *http.Request) (*layer.ListObjectVersionsParams, error) {
var (
err error
res layer.ListObjectVersionsParams
)
if r.URL.Query().Get("max-keys") == "" {
res.MaxKeys = maxObjectList
} else if res.MaxKeys, err = strconv.Atoi(r.URL.Query().Get("max-keys")); err != nil || res.MaxKeys <= 0 {
return nil, api.GetAPIError(api.ErrInvalidMaxKeys)
}
res.Prefix = r.URL.Query().Get("prefix")
res.KeyMarker = r.URL.Query().Get("marker")
res.Delimiter = r.URL.Query().Get("delimiter")
res.Encode = r.URL.Query().Get("encoding-type")
res.VersionIDMarker = r.URL.Query().Get("version-id-marker")
if info := api.GetReqInfo(r.Context()); info != nil {
res.Bucket = info.BucketName
}
return &res, nil
}
func encodeListObjectVersionsToResponse(info *layer.ListObjectVersionsInfo, bucketName string) *ListObjectsVersionsResponse {
res := ListObjectsVersionsResponse{
Name: bucketName,
IsTruncated: info.IsTruncated,
KeyMarker: info.KeyMarker,
NextKeyMarker: info.NextKeyMarker,
NextVersionIDMarker: info.NextVersionIDMarker,
VersionIDMarker: info.VersionIDMarker,
}
for _, prefix := range info.CommonPrefixes {
res.CommonPrefixes = append(res.CommonPrefixes, CommonPrefix{Prefix: *prefix})
}
for _, ver := range info.Version {
res.Version = append(res.Version, ObjectVersionResponse{
IsLatest: ver.IsLatest,
Key: ver.Object.Name,
LastModified: ver.Object.Created.Format(time.RFC3339),
Owner: Owner{
ID: ver.Object.Owner.String(),
DisplayName: ver.Object.Owner.String(),
},
Size: ver.Object.Size,
VersionID: ver.VersionID,
ETag: ver.Object.HashSum,
})
}
// this loop is not starting till versioning is not implemented
for _, del := range info.DeleteMarker {
res.DeleteMarker = append(res.DeleteMarker, DeleteMarkerEntry{
IsLatest: del.IsLatest,
Key: del.Key,
LastModified: del.LastModified,
Owner: Owner{
ID: del.Owner.String(),
DisplayName: del.Owner.String(),
},
VersionID: del.VersionID,
})
}
return &res
}

89
api/handler/response.go
View file

@ -14,35 +14,36 @@ type ListBucketsResponse struct {
} // Buckets are nested
}
// ListObjectsV2Response - format for list objects response.
// ListObjectsV1Response -- format for ListObjectsV1 response.
type ListObjectsV1Response struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"`
CommonPrefixes []CommonPrefix `xml:"CommonPrefixes"`
Contents []Object `xml:"Contents"`
Delimiter string `xml:"Delimiter,omitempty"`
EncodingType string `xml:"EncodingType,omitempty"`
IsTruncated bool `xml:"IsTruncated"`
Marker string `xml:"Marker"`
MaxKeys int `xml:"MaxKeys"`
Name string `xml:"Name"`
NextMarker string `xml:"NextMarker,omitempty"`
Prefix string `xml:"Prefix"`
}
// ListObjectsV2Response -- format for ListObjectsV2 response.
type ListObjectsV2Response struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"`
Name string
Prefix string
StartAfter string `xml:"StartAfter,omitempty"`
// When response is truncated (the IsTruncated element value in the response
// is true), you can use the key name in this field as marker in the subsequent
// request to get next set of objects. Server lists objects in alphabetical
// order Note: This element is returned only if you have delimiter request parameter
// specified. If response does not include the NextMaker and it is truncated,
// you can use the value of the last Key in the response as the marker in the
// subsequent request to get the next set of object keys.
ContinuationToken string `xml:"ContinuationToken,omitempty"`
NextContinuationToken string `xml:"NextContinuationToken,omitempty"`
KeyCount int
MaxKeys int
Delimiter string `xml:"Delimiter,omitempty"`
// A flag that indicates whether or not ListObjects returned all of the results
// that satisfied the search criteria.
IsTruncated bool
Contents []Object
CommonPrefixes []CommonPrefix
// Encoding type used to encode object keys in the response.
EncodingType string `xml:"EncodingType,omitempty"`
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"`
CommonPrefixes []CommonPrefix `xml:"CommonPrefixes"`
Contents []Object `xml:"Contents"`
ContinuationToken string `xml:"ContinuationToken,omitempty"`
Delimiter string `xml:"Delimiter,omitempty"`
EncodingType string `xml:"EncodingType,omitempty"`
IsTruncated bool `xml:"IsTruncated"`
KeyCount int `xml:"KeyCount"`
MaxKeys int `xml:"MaxKeys"`
Name string `xml:"Name"`
NextContinuationToken string `xml:"NextContinuationToken,omitempty"`
Prefix string `xml:"Prefix"`
StartAfter string `xml:"StartAfter,omitempty"`
}
// Bucket container for bucket metadata.
@ -57,37 +58,7 @@ type Owner struct {
DisplayName string
}
// ListObjectsResponse - format for list objects response.
type ListObjectsResponse struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"`
Name string
Prefix string
Marker string
// When response is truncated (the IsTruncated element value in the response
// is true), you can use the key name in this field as marker in the subsequent
// request to get next set of objects. Server lists objects in alphabetical
// order Note: This element is returned only if you have delimiter request parameter
// specified. If response does not include the NextMaker and it is truncated,
// you can use the value of the last Key in the response as the marker in the
// subsequent request to get the next set of object keys.
NextMarker string `xml:"NextMarker,omitempty"`
MaxKeys int
Delimiter string `xml:"Delimiter,omitempty"`
// A flag that indicates whether or not ListObjects returned all of the results
// that satisfied the search criteria.
IsTruncated bool
Contents []Object
CommonPrefixes []CommonPrefix
// Encoding type used to encode object keys in the response.
EncodingType string `xml:"EncodingType,omitempty"`
}
// CommonPrefix container for prefix response in ListObjectsResponse.
// CommonPrefix container for prefix response in ListObjects's response.
type CommonPrefix struct {
Prefix string
}

11
api/layer/container.go
View file

@ -28,17 +28,6 @@ type (
Owner *owner.ID
Created time.Time
}
// ListObjectsParams represents object listing request parameters.
ListObjectsParams struct {
Bucket string
Prefix string
Token string
Delimiter string
MaxKeys int
Marker string
Version int
}
)
func (n *layer) containerInfo(ctx context.Context, cid *cid.ID) (*BucketInfo, error) {

127
api/layer/layer.go
View file

@ -8,7 +8,6 @@ import (
"io"
"net/url"
"sort"
"strings"
"time"
"github.com/nspcc-dev/neofs-api-go/pkg/client"
@ -114,7 +113,8 @@ type (
CopyObject(ctx context.Context, p *CopyObjectParams) (*ObjectInfo, error)
ListObjects(ctx context.Context, p *ListObjectsParams) (*ListObjectsInfo, error)
ListObjectsV1(ctx context.Context, p *ListObjectsParamsV1) (*ListObjectsInfoV1, error)
ListObjectsV2(ctx context.Context, p *ListObjectsParamsV2) (*ListObjectsInfoV2, error)
ListObjectVersions(ctx context.Context, p *ListObjectVersionsParams) (*ListObjectVersionsInfo, error)
DeleteObject(ctx context.Context, bucket, object string) error
@ -134,8 +134,6 @@ var (
)
const (
// ETag (hex encoded md5sum) of empty string.
emptyETag = "d41d8cd98f00b204e9800998ecf8427e"
unversionedObjectVersionID = "null"
)
@ -212,127 +210,6 @@ func (n *layer) ListBuckets(ctx context.Context) ([]*BucketInfo, error) {
return n.containerList(ctx)
}
// ListObjects returns objects from the container. It ignores tombstones and
// storage groups.
// ctx, bucket, prefix, continuationToken, delimiter, maxKeys
func (n *layer) ListObjects(ctx context.Context, p *ListObjectsParams) (*ListObjectsInfo, error) {
// todo: make pagination when search response will be gRPC stream,
// pagination must be implemented with cache, because search results
// may be different between search calls
var (
err error
bkt *BucketInfo
ids []*object.ID
result ListObjectsInfo
uniqNames = make(map[string]bool)
)
if p.MaxKeys == 0 {
return &result, nil
}
if bkt, err = n.GetBucketInfo(ctx, p.Bucket); err != nil {
return nil, err
} else if ids, err = n.objectSearch(ctx, &findParams{cid: bkt.CID}); err != nil {
return nil, err
}
ln := len(ids)
// todo: check what happens if there is more than maxKeys objects
if ln > p.MaxKeys {
ln = p.MaxKeys
}
mostRecentModified := time.Time{}
needDirectoryAsKey := p.Version == 2 && len(p.Prefix) > 0 && len(p.Delimiter) > 0 && strings.HasSuffix(p.Prefix, p.Delimiter)
result.Objects = make([]*ObjectInfo, 0, ln)
for _, id := range ids {
addr := object.NewAddress()
addr.SetObjectID(id)
addr.SetContainerID(bkt.CID)
meta, err := n.objectHead(ctx, addr)
if err != nil {
n.log.Warn("could not fetch object meta", zap.Error(err))
continue
}
// // ignore tombstone objects
// _, hdr := meta.LastHeader(object.HeaderType(object.TombstoneHdr))
// if hdr != nil {
// continue
// }
// ignore storage group objects
// _, hdr = meta.LastHeader(object.HeaderType(object.StorageGroupHdr))
// if hdr != nil {
// continue
// }
// dirs don't exist in neofs, gateway stores full path to the file
// in object header, e.g. `filename`:`/this/is/path/file.txt`
// prefix argument contains full dir path from the root, e.g. `/this/is/`
// to emulate dirs we take dirs in path, compare it with prefix
// and look for entities after prefix. If entity does not have any
// sub-entities, then it is a file, else directory.
if oi := objectInfoFromMeta(bkt, meta, p.Prefix, p.Delimiter); oi != nil {
if needDirectoryAsKey && oi.Created.After(mostRecentModified) {
mostRecentModified = oi.Created
}
// use only unique dir names
if _, ok := uniqNames[oi.Name]; ok {
continue
}
if len(p.Marker) > 0 && oi.Name <= p.Marker {
continue
}
uniqNames[oi.Name] = oi.isDir
result.Objects = append(result.Objects, oi)
}
}
sort.Slice(result.Objects, func(i, j int) bool {
return result.Objects[i].Name < result.Objects[j].Name
})
if len(result.Objects) > p.MaxKeys {
result.IsTruncated = true
result.Objects = result.Objects[:p.MaxKeys]
result.NextMarker = result.Objects[len(result.Objects)-1].Name
}
fillPrefixes(&result, uniqNames)
if needDirectoryAsKey {
res := []*ObjectInfo{{
Name: p.Prefix,
Created: mostRecentModified,
HashSum: emptyETag,
}}
result.Objects = append(res, result.Objects...)
}
return &result, nil
}
func fillPrefixes(result *ListObjectsInfo, directories map[string]bool) {
index := 0
for range result.Objects {
name := result.Objects[index].Name
if isDir := directories[name]; isDir {
result.Objects = append(result.Objects[:index], result.Objects[index+1:]...)
result.Prefixes = append(result.Prefixes, name)
} else {
index++
}
}
}
// GetObject from storage.
func (n *layer) GetObject(ctx context.Context, p *GetObjectParams) error {
var (

44
api/layer/layer_test.go
View file

@ -1,44 +0,0 @@
package layer
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestFillingPrefixes(t *testing.T) {
cases := []struct {
name string
list *ListObjectsInfo
directories map[string]bool
expectedPrefixes []string
expectedObjects []*ObjectInfo
}{
{
name: "3 dirs",
list: &ListObjectsInfo{
Objects: []*ObjectInfo{{Name: "dir/"}, {Name: "dir2/"}, {Name: "dir3/"}},
},
directories: map[string]bool{"dir/": true, "dir2/": true, "dir3/": true},
expectedPrefixes: []string{"dir/", "dir2/", "dir3/"},
expectedObjects: []*ObjectInfo{},
},
{
name: "1 obj, 3 dirs",
list: &ListObjectsInfo{
Objects: []*ObjectInfo{{Name: "dir/"}, {Name: "dir2/"}, {Name: "dir3/"}, {Name: "obj"}},
},
directories: map[string]bool{"dir/": true, "dir2/": true, "dir3/": true},
expectedPrefixes: []string{"dir/", "dir2/", "dir3/"},
expectedObjects: []*ObjectInfo{{Name: "obj"}},
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
fillPrefixes(tc.list, tc.directories)
require.Equal(t, tc.expectedPrefixes, tc.list.Prefixes)
require.Equal(t, tc.expectedObjects, tc.list.Objects)
})
}
}

163
api/layer/object.go
View file

@ -5,6 +5,7 @@ import (
"errors"
"io"
"net/url"
"sort"
"strconv"
"time"
@ -12,6 +13,7 @@ import (
cid "github.com/nspcc-dev/neofs-api-go/pkg/container/id"
"github.com/nspcc-dev/neofs-api-go/pkg/object"
"github.com/nspcc-dev/neofs-s3-gw/api"
"go.uber.org/zap"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
@ -30,6 +32,35 @@ type (
length int64
address *object.Address
}
// ListObjectsParamsCommon contains common parameters for ListObjectsV1 and ListObjectsV2.
ListObjectsParamsCommon struct {
Bucket string
Delimiter string
Encode string
MaxKeys int
Prefix string
}
// ListObjectsParamsV1 contains params for ListObjectsV1.
ListObjectsParamsV1 struct {
ListObjectsParamsCommon
Marker string
}
// ListObjectsParamsV2 contains params for ListObjectsV2.
ListObjectsParamsV2 struct {
ListObjectsParamsCommon
ContinuationToken string
StartAfter string
}
allObjectParams struct {
Bucket string
Delimiter string
Prefix string
StartAfter string
}
)
// objectSearch returns all available objects by search params.
@ -170,3 +201,135 @@ func (n *layer) objectDelete(ctx context.Context, address *object.Address) error
dop.WithAddress(address)
return n.pool.DeleteObject(ctx, dop, n.BearerOpt(ctx))
}
// ListObjectsV1 returns objects in a bucket for requests of Version 1.
func (n *layer) ListObjectsV1(ctx context.Context, p *ListObjectsParamsV1) (*ListObjectsInfoV1, error) {
var (
err error
result ListObjectsInfoV1
)
if p.MaxKeys == 0 {
return &result, nil
}
allObjects, err := n.listSortedAllObjects(ctx, allObjectParams{
Bucket: p.Bucket,
Prefix: p.Prefix,
Delimiter: p.Delimiter,
StartAfter: p.Marker,
})
if err != nil {
return nil, err
}
if len(allObjects) > p.MaxKeys {
result.IsTruncated = true
nextObject := allObjects[p.MaxKeys-1]
result.NextMarker = nextObject.Name
allObjects = allObjects[:p.MaxKeys]
}
for _, ov := range allObjects {
if ov.isDir {
result.Prefixes = append(result.Prefixes, ov.Name)
} else {
result.Objects = append(result.Objects, ov)
}
}
return &result, nil
}
// ListObjectsV2 returns objects in a bucket for requests of Version 2.
func (n *layer) ListObjectsV2(ctx context.Context, p *ListObjectsParamsV2) (*ListObjectsInfoV2, error) {
var (
err error
result ListObjectsInfoV2
allObjects []*ObjectInfo
)
if p.MaxKeys == 0 {
return &result, nil
}
if p.ContinuationToken != "" {
// find cache with continuation token
} else {
allObjects, err = n.listSortedAllObjects(ctx, allObjectParams{
Bucket: p.Bucket,
Prefix: p.Prefix,
Delimiter: p.Delimiter,
StartAfter: p.StartAfter,
})
if err != nil {
return nil, err
}
}
if len(allObjects) > p.MaxKeys {
result.IsTruncated = true
allObjects = allObjects[:p.MaxKeys]
// add creating of cache here
}
for _, ov := range allObjects {
if ov.isDir {
result.Prefixes = append(result.Prefixes, ov.Name)
} else {
result.Objects = append(result.Objects, ov)
}
}
return &result, nil
}
func (n *layer) listSortedAllObjects(ctx context.Context, p allObjectParams) ([]*ObjectInfo, error) {
var (
err error
bkt *BucketInfo
ids []*object.ID
uniqNames = make(map[string]bool)
)
if bkt, err = n.GetBucketInfo(ctx, p.Bucket); err != nil {
return nil, err
} else if ids, err = n.objectSearch(ctx, &findParams{cid: bkt.CID}); err != nil {
return nil, err
}
objects := make([]*ObjectInfo, 0, len(ids))
for _, id := range ids {
addr := object.NewAddress()
addr.SetObjectID(id)
addr.SetContainerID(bkt.CID)
meta, err := n.objectHead(ctx, addr)
if err != nil {
n.log.Warn("could not fetch object meta", zap.Error(err))
continue
}
if oi := objectInfoFromMeta(bkt, meta, p.Prefix, p.Delimiter); oi != nil {
// use only unique dir names
if _, ok := uniqNames[oi.Name]; ok {
continue
}
if len(p.StartAfter) > 0 && oi.Name <= p.StartAfter {
continue
}
uniqNames[oi.Name] = oi.isDir
objects = append(objects, oi)
}
}
sort.Slice(objects, func(i, j int) bool {
return objects[i].Name < objects[j].Name
})
return objects, nil
}

33
api/layer/util.go
View file

@ -26,32 +26,23 @@ type (
Headers map[string]string
}
// ListObjectsInfo - container for list objects.
// ListObjectsInfo contains common fields of data for ListObjectsV1 and ListObjectsV2.
ListObjectsInfo struct {
// Indicates whether the returned list objects response is truncated. A
// value of true indicates that the list was truncated. The list can be truncated
// if the number of objects exceeds the limit allowed or specified
// by max keys.
Prefixes []string
Objects []*ObjectInfo
IsTruncated bool
}
// When response is truncated (the IsTruncated element value in the response
// is true), you can use the key name in this field as marker in the subsequent
// request to get next set of objects.
//
// NOTE: This element is returned only if you have delimiter request parameter
// specified.
ContinuationToken string
NextContinuationToken string
// When response is truncated (the IsTruncated element value in the response is true),
// you can use the key name in this field as marker in the subsequent request to get next set of objects.
// ListObjectsInfoV1 holds data which ListObjectsV1 returns.
ListObjectsInfoV1 struct {
ListObjectsInfo
NextMarker string
}
// List of objects info for this request.
Objects []*ObjectInfo
// List of prefixes for this request.
Prefixes []string
// ListObjectsInfoV2 holds data which ListObjectsV2 returns.
ListObjectsInfoV2 struct {
ListObjectsInfo
NextContinuationToken string
}
// ObjectVersionInfo stores info about objects versions.