Merge pull request #30 from nspcc-dev/api-handlers

Refactoring API handlers
This commit is contained in:
Evgeniy Kulikov 2020-08-23 03:10:04 +03:00 committed by GitHub
commit 96ffa4d26c
12 changed files with 359 additions and 63 deletions

View file

@ -1626,14 +1626,23 @@ func GetAPIError(code ErrorCode) Error {
// getErrorResponse gets in standard error and resource value and
// provides a encodable populated response values
func getAPIErrorResponse(ctx context.Context, err Error, resource, requestID, hostID string) ErrorResponse {
func getAPIErrorResponse(ctx context.Context, err error, resource, requestID, hostID string) ErrorResponse {
code := "BadRequest"
desc := err.Error()
info := GetReqInfo(ctx)
if info == nil {
info = &ReqInfo{}
}
if e, ok := err.(Error); ok {
code = e.Code
desc = e.Description
}
return ErrorResponse{
Code: err.Code,
Message: err.Description,
Code: code,
Message: desc,
BucketName: info.BucketName,
Key: info.ObjectName,
Resource: resource,
@ -2000,3 +2009,13 @@ type PreConditionFailed struct{}
func (e PreConditionFailed) Error() string {
return "At least one of the pre-conditions you specified did not hold"
}
// DeleteError - returns when cant remove object
type DeleteError struct {
Err error
Object string
}
func (e DeleteError) Error() string {
return fmt.Sprintf("%s (%s)", e.Err, e.Object)
}

View file

@ -1,13 +1,46 @@
package handler
import (
"encoding/xml"
"net/http"
"github.com/gorilla/mux"
"github.com/nspcc-dev/neofs-s3-gate/api"
"go.uber.org/zap"
"google.golang.org/grpc/status"
)
// DeleteObjectsRequest - xml carrying the object key names which needs to be deleted.
type DeleteObjectsRequest struct {
// Element to enable quiet mode for the request
Quiet bool
// List of objects to be deleted
Objects []ObjectIdentifier `xml:"Object"`
}
// ObjectIdentifier carries key name for the object to delete.
type ObjectIdentifier struct {
ObjectName string `xml:"Key"`
}
// DeleteError structure.
type DeleteError struct {
Code string
Message string
Key string
}
// DeleteObjectsResponse container for multiple object deletes.
type DeleteObjectsResponse struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ DeleteResult" json:"-"`
// Collection of all deleted objects
DeletedObjects []ObjectIdentifier `xml:"Deleted,omitempty"`
// Collection of errors deleting certain objects.
Errors []DeleteError `xml:"Error,omitempty"`
}
func (h *handler) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) {
var (
req = mux.Vars(r)
@ -36,13 +69,89 @@ func (h *handler) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) {
}
// DeleteMultipleObjectsHandler :
//
// CyberDuck doesn't use that method for multiple delete.
// Open issue and describe how to test that method.
func (h *handler) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) {
api.WriteErrorResponse(r.Context(), w, api.Error{
Code: "XNeoFSUnimplemented",
Description: "implement me " + mux.CurrentRoute(r).GetName(),
HTTPStatusCode: http.StatusNotImplemented,
}, r.URL)
var (
req = mux.Vars(r)
bkt = req["bucket"]
rid = api.GetRequestID(r.Context())
)
// Content-Md5 is requied should be set
// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
if _, ok := r.Header[api.ContentMD5]; !ok {
api.WriteErrorResponse(r.Context(), w, api.GetAPIError(api.ErrMissingContentMD5), r.URL)
return
}
// Content-Length is required and should be non-zero
// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
if r.ContentLength <= 0 {
api.WriteErrorResponse(r.Context(), w, api.GetAPIError(api.ErrMissingContentLength), r.URL)
return
}
// Unmarshal list of keys to be deleted.
requested := &DeleteObjectsRequest{}
if err := xml.NewDecoder(r.Body).Decode(requested); err != nil {
api.WriteErrorResponse(r.Context(), w, err, r.URL)
return
}
removed := make(map[string]struct{})
toRemove := make([]string, 0, len(requested.Objects))
for _, obj := range requested.Objects {
removed[obj.ObjectName] = struct{}{}
toRemove = append(toRemove, obj.ObjectName)
}
response := &DeleteObjectsResponse{
Errors: make([]DeleteError, 0, len(toRemove)),
DeletedObjects: make([]ObjectIdentifier, 0, len(toRemove)),
}
if errs := h.obj.DeleteObjects(r.Context(), bkt, toRemove); errs != nil && !requested.Quiet {
h.log.Error("could not delete objects",
zap.String("request_id", rid),
zap.String("bucket_name", bkt),
zap.Strings("object_name", toRemove),
zap.Errors("errors", errs))
for _, e := range errs {
if err, ok := e.(*api.DeleteError); ok {
code := "BadRequest"
desc := err.Error()
if st, ok := status.FromError(err.Err); ok && st != nil {
desc = st.Message()
code = st.Code().String()
}
response.Errors = append(response.Errors, DeleteError{
Code: code,
Message: desc,
Key: err.Object,
})
delete(removed, err.Object)
}
}
}
for key := range removed {
response.DeletedObjects = append(response.DeletedObjects, ObjectIdentifier{ObjectName: key})
}
if err := api.EncodeToResponse(w, response); err != nil {
h.log.Error("could not write response",
zap.String("request_id", rid),
zap.String("bucket_name", bkt),
zap.Strings("object_name", toRemove),
zap.Error(err))
api.WriteErrorResponse(r.Context(), w, api.Error{
Code: api.GetAPIError(api.ErrInternalError).Code,
Description: err.Error(),
HTTPStatusCode: http.StatusInternalServerError,
}, r.URL)
}
}

View file

@ -8,6 +8,8 @@ import (
"github.com/nspcc-dev/neofs-s3-gate/api"
"github.com/nspcc-dev/neofs-s3-gate/api/layer"
"go.uber.org/zap"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
func (h *handler) HeadObjectHandler(w http.ResponseWriter, r *http.Request) {
@ -45,3 +47,34 @@ func (h *handler) HeadObjectHandler(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Last-Modified", inf.Created.Format(http.TimeFormat))
}
func (h *handler) HeadBucketHandler(w http.ResponseWriter, r *http.Request) {
var (
req = mux.Vars(r)
bkt = req["bucket"]
rid = api.GetRequestID(r.Context())
)
if _, err := h.obj.GetBucketInfo(r.Context(), bkt); err != nil {
h.log.Error("could not fetch object info",
zap.String("request_id", rid),
zap.String("bucket_name", bkt),
zap.Error(err))
code := http.StatusBadRequest
if st, ok := status.FromError(err); ok && st != nil {
switch st.Code() {
case codes.NotFound:
code = http.StatusNotFound
case codes.PermissionDenied:
code = http.StatusForbidden
}
}
api.WriteResponse(w, code, nil, api.MimeNone)
return
}
api.WriteResponse(w, http.StatusOK, nil, api.MimeNone)
}

View file

@ -86,11 +86,10 @@ func (h *handler) ListBucketsHandler(w http.ResponseWriter, r *http.Request) {
}
}
func (h *handler) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {
func (h *handler) listObjects(w http.ResponseWriter, r *http.Request) (*listObjectsArgs, *layer.ListObjectsInfo, error) {
var (
err error
arg *listObjectsArgs
res *ListObjectsResponse
rid = api.GetRequestID(r.Context())
)
@ -105,7 +104,7 @@ func (h *handler) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {
HTTPStatusCode: http.StatusBadRequest,
}, r.URL)
return
return nil, nil, err
}
list, err := h.obj.ListObjects(r.Context(), &layer.ListObjectsParams{
@ -125,10 +124,32 @@ func (h *handler) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {
HTTPStatusCode: http.StatusInternalServerError,
}, r.URL)
return
return nil, nil, err
}
res = &ListObjectsResponse{
return arg, list, nil
}
func (h *handler) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {
var rid = api.GetRequestID(r.Context())
if arg, list, err := h.listObjects(w, r); err != nil {
// error already sent to client
return
} else if err := api.EncodeToResponse(w, encodeV1(arg, list)); err != nil {
h.log.Error("something went wrong",
zap.String("request_id", rid),
zap.Error(err))
api.WriteErrorResponse(r.Context(), w, api.Error{
Code: api.GetAPIError(api.ErrInternalError).Code,
Description: err.Error(),
HTTPStatusCode: http.StatusInternalServerError,
}, r.URL)
}
}
func encodeV1(arg *listObjectsArgs, list *layer.ListObjectsInfo) *ListObjectsResponse {
res := &ListObjectsResponse{
Name: arg.Bucket,
EncodingType: arg.Encode,
Marker: arg.Marker,
@ -155,13 +176,25 @@ func (h *handler) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {
UserMetadata: obj.Headers,
LastModified: obj.Created.Format(time.RFC3339),
Owner: Owner{
ID: obj.Owner.String(),
DisplayName: obj.Owner.String(),
},
// ETag: "",
// Owner: Owner{},
// StorageClass: "",
})
}
if err := api.EncodeToResponse(w, res); err != nil {
return res
}
func (h *handler) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) {
var rid = api.GetRequestID(r.Context())
if arg, list, err := h.listObjects(w, r); err != nil {
// error already sent to client
return
} else if err := api.EncodeToResponse(w, encodeV2(arg, list)); err != nil {
h.log.Error("something went wrong",
zap.String("request_id", rid),
zap.Error(err))
@ -174,6 +207,48 @@ func (h *handler) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {
}
}
func encodeV2(arg *listObjectsArgs, list *layer.ListObjectsInfo) *ListObjectsV2Response {
res := &ListObjectsV2Response{
Name: arg.Bucket,
EncodingType: arg.Encode,
Prefix: arg.Prefix,
MaxKeys: arg.MaxKeys,
Delimiter: arg.Delimeter,
IsTruncated: list.IsTruncated,
ContinuationToken: arg.Marker,
NextContinuationToken: list.NextContinuationToken,
}
// fill common prefixes
for i := range list.Prefixes {
res.CommonPrefixes = append(res.CommonPrefixes, CommonPrefix{
Prefix: list.Prefixes[i],
})
}
// fill contents
for _, obj := range list.Objects {
res.Contents = append(res.Contents, Object{
Key: obj.Name,
Size: obj.Size,
UserMetadata: obj.Headers,
LastModified: obj.Created.Format(time.RFC3339),
Owner: Owner{
ID: obj.Owner.String(),
DisplayName: obj.Owner.String(),
},
// ETag: "",
// StorageClass: "",
})
}
return res
}
func parseListObjectArgs(r *http.Request) (*listObjectsArgs, error) {
var (
err error

View file

@ -14,6 +14,37 @@ type ListBucketsResponse struct {
} // Buckets are nested
}
// ListObjectsV2Response - format for list objects response.
type ListObjectsV2Response struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"`
Name string
Prefix string
StartAfter string `xml:"StartAfter,omitempty"`
// When response is truncated (the IsTruncated element value in the response
// is true), you can use the key name in this field as marker in the subsequent
// request to get next set of objects. Server lists objects in alphabetical
// order Note: This element is returned only if you have delimiter request parameter
// specified. If response does not include the NextMaker and it is truncated,
// you can use the value of the last Key in the response as the marker in the
// subsequent request to get the next set of object keys.
ContinuationToken string `xml:"ContinuationToken,omitempty"`
NextContinuationToken string `xml:"NextContinuationToken,omitempty"`
KeyCount int
MaxKeys int
Delimiter string
// A flag that indicates whether or not ListObjects returned all of the results
// that satisfied the search criteria.
IsTruncated bool
Contents []Object
CommonPrefixes []CommonPrefix
// Encoding type used to encode object keys in the response.
EncodingType string `xml:"EncodingType,omitempty"`
}
// Bucket container for bucket metadata
type Bucket struct {
Name string

View file

@ -295,14 +295,6 @@ func (h *handler) ListObjectsV2MHandler(w http.ResponseWriter, r *http.Request)
}, r.URL)
}
func (h *handler) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) {
api.WriteErrorResponse(r.Context(), w, api.Error{
Code: "XNeoFSUnimplemented",
Description: "implement me " + mux.CurrentRoute(r).GetName(),
HTTPStatusCode: http.StatusNotImplemented,
}, r.URL)
}
func (h *handler) ListBucketObjectVersionsHandler(w http.ResponseWriter, r *http.Request) {
api.WriteErrorResponse(r.Context(), w, api.Error{
Code: "XNeoFSUnimplemented",
@ -343,14 +335,6 @@ func (h *handler) PutBucketObjectLockConfigHandler(w http.ResponseWriter, r *htt
}, r.URL)
}
func (h *handler) HeadBucketHandler(w http.ResponseWriter, r *http.Request) {
api.WriteErrorResponse(r.Context(), w, api.Error{
Code: "XNeoFSUnimplemented",
Description: "implement me " + mux.CurrentRoute(r).GetName(),
HTTPStatusCode: http.StatusNotImplemented,
}, r.URL)
}
func (h *handler) PostPolicyBucketHandler(w http.ResponseWriter, r *http.Request) {
api.WriteErrorResponse(r.Context(), w, api.Error{
Code: "XNeoFSUnimplemented",

25
api/headers.go Normal file
View file

@ -0,0 +1,25 @@
package api
// Standard S3 HTTP response constants
const (
LastModified = "Last-Modified"
Date = "Date"
ETag = "ETag"
ContentType = "Content-Type"
ContentMD5 = "Content-Md5"
ContentEncoding = "Content-Encoding"
Expires = "Expires"
ContentLength = "Content-Length"
ContentLanguage = "Content-Language"
ContentRange = "Content-Range"
Connection = "Connection"
AcceptRanges = "Accept-Ranges"
AmzBucketRegion = "X-Amz-Bucket-Region"
ServerInfo = "Server"
RetryAfter = "Retry-After"
Location = "Location"
CacheControl = "Cache-Control"
ContentDisposition = "Content-Disposition"
Authorization = "Authorization"
Action = "Action"
)

View file

@ -12,8 +12,9 @@ import (
"github.com/nspcc-dev/neofs-api-go/service"
"github.com/nspcc-dev/neofs-s3-gate/api"
"github.com/nspcc-dev/neofs-s3-gate/api/pool"
"github.com/pkg/errors"
"go.uber.org/zap"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
type (
@ -68,7 +69,7 @@ type (
ListObjects(ctx context.Context, p *ListObjectsParams) (*ListObjectsInfo, error)
DeleteObject(ctx context.Context, bucket, object string) error
DeleteObjects(ctx context.Context, bucket string, objects []string) ([]error, error)
DeleteObjects(ctx context.Context, bucket string, objects []string) []error
}
)
@ -142,7 +143,7 @@ func (n *layer) GetBucketInfo(ctx context.Context, name string) (*BucketInfo, er
}
}
return nil, errors.New("bucket not found")
return nil, status.Error(codes.NotFound, "bucket not found")
}
// ListBuckets returns all user containers. Name of the bucket is a container
@ -224,6 +225,7 @@ func (n *layer) ListObjects(ctx context.Context, p *ListObjectsParams) (*ListObj
oi = objectInfoFromMeta(meta)
} else { // if there are sub-entities in tail - dir
oi = &ObjectInfo{
Owner: meta.SystemHeader.OwnerID,
Bucket: meta.SystemHeader.CID.String(),
Name: tail[:ind+1], // dir MUST have slash symbol in the end
// IsDir: true,
@ -375,17 +377,26 @@ func (n *layer) CopyObject(ctx context.Context, p *CopyObjectParams) (*ObjectInf
func (n *layer) DeleteObject(ctx context.Context, bucket, object string) error {
cid, err := refs.CIDFromString(bucket)
if err != nil {
return err
return &api.DeleteError{
Err: err,
Object: object,
}
}
ids, err := n.objectFindIDs(ctx, cid, object)
if err != nil {
return errors.Wrap(err, "could not find object")
return &api.DeleteError{
Err: err,
Object: object,
}
}
for _, id := range ids {
if err = n.objectDelete(ctx, delParams{addr: refs.Address{CID: cid, ObjectID: id}}); err != nil {
return errors.Wrapf(err, "could not remove object: %s => %s", object, id)
return &api.DeleteError{
Err: err,
Object: object,
}
}
}
@ -393,12 +404,14 @@ func (n *layer) DeleteObject(ctx context.Context, bucket, object string) error {
}
// DeleteObjects from the storage.
func (n *layer) DeleteObjects(ctx context.Context, bucket string, objects []string) ([]error, error) {
func (n *layer) DeleteObjects(ctx context.Context, bucket string, objects []string) []error {
var errs = make([]error, 0, len(objects))
for i := range objects {
errs = append(errs, n.DeleteObject(ctx, bucket, objects[i]))
if err := n.DeleteObject(ctx, bucket, objects[i]); err != nil {
errs = append(errs, err)
}
}
return errs, nil
return errs
}

View file

@ -13,6 +13,8 @@ import (
"github.com/nspcc-dev/neofs-s3-gate/api/pool"
"github.com/pkg/errors"
"go.uber.org/zap"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
const (
@ -226,7 +228,7 @@ func (n *layer) objectFindID(ctx context.Context, cid refs.CID, name string, put
return id, nil
}
}
return id, errors.New("object not found")
return id, status.Error(codes.NotFound, "object not found")
} else if ln == 1 {
return result[0], nil
}

View file

@ -7,6 +7,7 @@ import (
"time"
"github.com/nspcc-dev/neofs-api-go/object"
"github.com/nspcc-dev/neofs-api-go/refs"
)
type (
@ -16,6 +17,7 @@ type (
Size int64
ContentType string
Created time.Time
Owner refs.OwnerID
Headers map[string]string
}

View file

@ -118,21 +118,25 @@ var s3ErrorResponseMap = map[string]string{
}
// WriteErrorResponse writes error headers
func WriteErrorResponse(ctx context.Context, w http.ResponseWriter, err Error, reqURL *url.URL) {
switch err.Code {
case "SlowDown", "XNeoFSServerNotInitialized", "XNeoFSReadQuorum", "XNeoFSWriteQuorum":
// Set retry-after header to indicate user-agents to retry request after 120secs.
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
w.Header().Set(hdrRetryAfter, "120")
case "AccessDenied":
// TODO process when the request is from browser and also if browser
func WriteErrorResponse(ctx context.Context, w http.ResponseWriter, err error, reqURL *url.URL) {
code := http.StatusBadRequest
if e, ok := err.(Error); ok {
switch e.Code {
case "SlowDown", "XNeoFSServerNotInitialized", "XNeoFSReadQuorum", "XNeoFSWriteQuorum":
// Set retry-after header to indicate user-agents to retry request after 120secs.
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
w.Header().Set(hdrRetryAfter, "120")
case "AccessDenied":
// TODO process when the request is from browser and also if browser
}
}
// Generate error response.
errorResponse := getAPIErrorResponse(ctx, err, reqURL.Path,
w.Header().Get(hdrAmzRequestID), deploymentID.String())
encodedErrorResponse := EncodeResponse(errorResponse)
writeResponse(w, err.HTTPStatusCode, encodedErrorResponse, mimeXML)
WriteResponse(w, code, encodedErrorResponse, MimeXML)
}
// If none of the http routes match respond with appropriate errors
@ -162,9 +166,9 @@ func removeSensitiveHeaders(h http.Header) {
h.Del(hdrSSECopyKey)
}
func writeResponse(w http.ResponseWriter, statusCode int, response []byte, mType mimeType) {
func WriteResponse(w http.ResponseWriter, statusCode int, response []byte, mType mimeType) {
setCommonHeaders(w)
if mType != mimeNone {
if mType != MimeNone {
w.Header().Set(hdrContentType, string(mType))
}
w.Header().Set(hdrContentLength, strconv.Itoa(len(response)))
@ -196,11 +200,11 @@ func EncodeToResponse(w http.ResponseWriter, response interface{}) error {
// WriteSuccessResponseXML writes success headers and response if any,
// with content-type set to `application/xml`.
func WriteSuccessResponseXML(w http.ResponseWriter, response []byte) {
writeResponse(w, http.StatusOK, response, mimeXML)
WriteResponse(w, http.StatusOK, response, MimeXML)
}
func WriteSuccessResponseHeadersOnly(w http.ResponseWriter) {
writeResponse(w, http.StatusOK, nil, mimeNone)
WriteResponse(w, http.StatusOK, nil, MimeNone)
}
// Error - Returns S3 error string.

View file

@ -90,12 +90,11 @@ const (
// SlashSeparator - slash separator.
SlashSeparator = "/"
// Means no response type.
mimeNone mimeType = ""
// Means response type is JSON.
// mimeJSON mimeType = "application/json"
// Means response type is XML.
mimeXML mimeType = "application/xml"
// MimeNone means no response type.
MimeNone mimeType = ""
// MimeXML means response type is XML.
MimeXML mimeType = "application/xml"
)
var _ = logErrorResponse