diff --git a/api/handler/bucket_list.go b/api/handler/bucket_list.go new file mode 100644 index 00000000..e060b676 --- /dev/null +++ b/api/handler/bucket_list.go @@ -0,0 +1,70 @@ +package handler + +import ( + "net/http" + "strconv" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware" +) + +const maxBucketList = 10000 + +// ListBucketsHandler handles bucket listing requests. +func (h *handler) ListBucketsHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + reqInfo := middleware.GetReqInfo(ctx) + + params, err := parseListBucketParams(r) + if err != nil { + h.logAndSendError(ctx, w, "failed to parse params", reqInfo, err) + return + } + + resp, err := h.obj.ListBuckets(ctx, params) + if err != nil { + h.logAndSendError(ctx, w, "something went wrong", reqInfo, err) + return + } + + if err = middleware.EncodeToResponse(w, encodeListBuckets(reqInfo.User, resp, params)); err != nil { + h.logAndSendError(ctx, w, "something went wrong", reqInfo, err) + } +} + +func encodeListBuckets(owner string, resp layer.ListBucketsResult, params layer.ListBucketsParams) *ListBucketsResponse { + res := &ListBucketsResponse{ + Owner: Owner{ + ID: owner, + DisplayName: owner, + }, + ContinuationToken: resp.ContinuationToken, + Prefix: params.Prefix, + } + + for _, item := range resp.Containers { + res.Buckets.Buckets = append(res.Buckets.Buckets, Bucket{ + Name: item.Name, + CreationDate: item.Created.UTC().Format(time.RFC3339), + BucketRegion: item.LocationConstraint, + }) + } + return res +} + +func parseListBucketParams(r *http.Request) (prm layer.ListBucketsParams, err error) { + prm.MaxBuckets = maxBucketList + strMaxBuckets := r.URL.Query().Get(middleware.QueryMaxBuckets) + if strMaxBuckets != "" { + if prm.MaxBuckets, err = strconv.Atoi(strMaxBuckets); err != nil || prm.MaxBuckets < 0 { + return layer.ListBucketsParams{}, errors.GetAPIError(errors.ErrInvalidMaxKeys) + } + } + prm.Prefix = r.URL.Query().Get(middleware.QueryPrefix) + prm.BucketRegion = r.URL.Query().Get(middleware.QueryBucketRegion) + prm.ContinuationToken = r.URL.Query().Get(middleware.QueryContinuationToken) + + return +} diff --git a/api/handler/list.go b/api/handler/list.go deleted file mode 100644 index f6bfae3a..00000000 --- a/api/handler/list.go +++ /dev/null @@ -1,49 +0,0 @@ -package handler - -import ( - "net/http" - "time" - - "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" -) - -const maxObjectList = 1000 // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse. - -// ListBucketsHandler handles bucket listing requests. -func (h *handler) ListBucketsHandler(w http.ResponseWriter, r *http.Request) { - var ( - own user.ID - res *ListBucketsResponse - ctx = r.Context() - reqInfo = middleware.GetReqInfo(ctx) - ) - - list, err := h.obj.ListBuckets(ctx) - if err != nil { - h.logAndSendError(ctx, w, "something went wrong", reqInfo, err) - return - } - - if len(list) > 0 { - own = list[0].Owner - } - - res = &ListBucketsResponse{ - Owner: Owner{ - ID: own.String(), - DisplayName: own.String(), - }, - } - - for _, item := range list { - res.Buckets.Buckets = append(res.Buckets.Buckets, Bucket{ - Name: item.Name, - CreationDate: item.Created.UTC().Format(time.RFC3339), - }) - } - - if err = middleware.EncodeToResponse(w, res); err != nil { - h.logAndSendError(ctx, w, "something went wrong", reqInfo, err) - } -} diff --git a/api/handler/object_list.go b/api/handler/object_list.go index 7c20f37d..33183966 100644 --- a/api/handler/object_list.go +++ b/api/handler/object_list.go @@ -15,6 +15,8 @@ import ( oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) +const maxObjectList = 1000 // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse. + // ListObjectsV1Handler handles objects listing requests for API version 1. func (h *handler) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) { ctx := r.Context() diff --git a/api/handler/response.go b/api/handler/response.go index 8654d8d9..eb21b2af 100644 --- a/api/handler/response.go +++ b/api/handler/response.go @@ -15,6 +15,9 @@ type ListBucketsResponse struct { Buckets struct { Buckets []Bucket `xml:"Bucket"` } // Buckets are nested + + ContinuationToken string `xml:"ContinuationToken,omitempty"` + Prefix string `xml:"Prefix,omitempty"` } // ListObjectsV1Response -- format for ListObjectsV1 response. @@ -51,8 +54,9 @@ type ListObjectsV2Response struct { // Bucket container for bucket metadata. type Bucket struct { - Name string - CreationDate string // time string of format "2006-01-02T15:04:05.000Z" + Name string `xml:"Name"` + CreationDate string `xml:"CreationDate"` // time string of format "2006-01-02T15:04:05.000Z" + BucketRegion string `xml:"BucketRegion,omitempty"` } // PolicyStatus contains status of bucket policy. diff --git a/api/layer/container.go b/api/layer/container.go index ebb23d76..cc9346eb 100644 --- a/api/layer/container.go +++ b/api/layer/container.go @@ -3,7 +3,9 @@ package layer import ( "context" "fmt" + "sort" "strconv" + "strings" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data" @@ -76,7 +78,7 @@ func (n *Layer) containerInfo(ctx context.Context, prm frostfs.PrmContainer) (*d return info, nil } -func (n *Layer) containerList(ctx context.Context) ([]*data.BucketInfo, error) { +func (n *Layer) containerList(ctx context.Context, listParams ListBucketsParams) ([]*data.BucketInfo, error) { stoken := n.SessionTokenForRead(ctx) prm := frostfs.PrmUserContainers{ @@ -102,10 +104,34 @@ func (n *Layer) containerList(ctx context.Context) ([]*data.BucketInfo, error) { continue } + if shouldSkipBucket(info, listParams) { + continue + } + list = append(list, info) } - return list, nil + sort.Slice(list, func(i, j int) bool { + return list[i].Name < list[j].Name + }) + + for i, info := range list { + if listParams.ContinuationToken != "" && info.Name != listParams.ContinuationToken { + continue + } + return list[i:], nil + } + + return nil, nil +} + +func shouldSkipBucket(info *data.BucketInfo, prm ListBucketsParams) bool { + if !strings.HasPrefix(info.Name, prm.Prefix) || + (prm.BucketRegion != "" && info.LocationConstraint != prm.BucketRegion) { + return true + } + + return false } func (n *Layer) createContainer(ctx context.Context, p *CreateBucketParams) (*data.BucketInfo, error) { diff --git a/api/layer/layer.go b/api/layer/layer.go index bd7e7d8c..7b2d8ec3 100644 --- a/api/layer/layer.go +++ b/api/layer/layer.go @@ -195,6 +195,18 @@ type ( Encode string } + ListBucketsParams struct { + MaxBuckets int + Prefix string + ContinuationToken string + BucketRegion string + } + + ListBucketsResult struct { + Containers []*data.BucketInfo + ContinuationToken string + } + // VersionedObject stores info about objects to delete. VersionedObject struct { Name string @@ -371,8 +383,24 @@ func (n *Layer) ResolveCID(ctx context.Context, name string) (cid.ID, error) { // ListBuckets returns all user containers. The name of the bucket is a container // id. Timestamp is omitted since it is not saved in frostfs container. -func (n *Layer) ListBuckets(ctx context.Context) ([]*data.BucketInfo, error) { - return n.containerList(ctx) +func (n *Layer) ListBuckets(ctx context.Context, params ListBucketsParams) (ListBucketsResult, error) { + var result ListBucketsResult + var err error + + if params.MaxBuckets == 0 { + return result, nil + } + + result.Containers, err = n.containerList(ctx, params) + if err != nil { + return ListBucketsResult{}, err + } + if len(result.Containers) > params.MaxBuckets { + result.ContinuationToken = result.Containers[params.MaxBuckets].Name + result.Containers = result.Containers[:params.MaxBuckets] + } + + return result, nil } // GetObject from storage.