diff --git a/api/errors.go b/api/errors.go index 86906141..a0e6e39b 100644 --- a/api/errors.go +++ b/api/errors.go @@ -1626,14 +1626,23 @@ func GetAPIError(code ErrorCode) Error { // getErrorResponse gets in standard error and resource value and // provides a encodable populated response values -func getAPIErrorResponse(ctx context.Context, err Error, resource, requestID, hostID string) ErrorResponse { +func getAPIErrorResponse(ctx context.Context, err error, resource, requestID, hostID string) ErrorResponse { + code := "BadRequest" + desc := err.Error() + info := GetReqInfo(ctx) if info == nil { info = &ReqInfo{} } + + if e, ok := err.(Error); ok { + code = e.Code + desc = e.Description + } + return ErrorResponse{ - Code: err.Code, - Message: err.Description, + Code: code, + Message: desc, BucketName: info.BucketName, Key: info.ObjectName, Resource: resource, @@ -2000,3 +2009,13 @@ type PreConditionFailed struct{} func (e PreConditionFailed) Error() string { return "At least one of the pre-conditions you specified did not hold" } + +// DeleteError - returns when cant remove object +type DeleteError struct { + Err error + Object string +} + +func (e DeleteError) Error() string { + return fmt.Sprintf("%s (%s)", e.Err, e.Object) +} diff --git a/api/handler/delete.go b/api/handler/delete.go index 11daaa80..2e70d94c 100644 --- a/api/handler/delete.go +++ b/api/handler/delete.go @@ -1,13 +1,46 @@ package handler import ( + "encoding/xml" "net/http" "github.com/gorilla/mux" "github.com/nspcc-dev/neofs-s3-gate/api" "go.uber.org/zap" + "google.golang.org/grpc/status" ) +// DeleteObjectsRequest - xml carrying the object key names which needs to be deleted. +type DeleteObjectsRequest struct { + // Element to enable quiet mode for the request + Quiet bool + // List of objects to be deleted + Objects []ObjectIdentifier `xml:"Object"` +} + +// ObjectIdentifier carries key name for the object to delete. +type ObjectIdentifier struct { + ObjectName string `xml:"Key"` +} + +// DeleteError structure. +type DeleteError struct { + Code string + Message string + Key string +} + +// DeleteObjectsResponse container for multiple object deletes. +type DeleteObjectsResponse struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ DeleteResult" json:"-"` + + // Collection of all deleted objects + DeletedObjects []ObjectIdentifier `xml:"Deleted,omitempty"` + + // Collection of errors deleting certain objects. + Errors []DeleteError `xml:"Error,omitempty"` +} + func (h *handler) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) { var ( req = mux.Vars(r) @@ -36,13 +69,89 @@ func (h *handler) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) { } // DeleteMultipleObjectsHandler : -// -// CyberDuck doesn't use that method for multiple delete. -// Open issue and describe how to test that method. func (h *handler) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) { - api.WriteErrorResponse(r.Context(), w, api.Error{ - Code: "XNeoFSUnimplemented", - Description: "implement me " + mux.CurrentRoute(r).GetName(), - HTTPStatusCode: http.StatusNotImplemented, - }, r.URL) + var ( + req = mux.Vars(r) + bkt = req["bucket"] + rid = api.GetRequestID(r.Context()) + ) + + // Content-Md5 is requied should be set + // http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html + if _, ok := r.Header[api.ContentMD5]; !ok { + api.WriteErrorResponse(r.Context(), w, api.GetAPIError(api.ErrMissingContentMD5), r.URL) + return + } + + // Content-Length is required and should be non-zero + // http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html + if r.ContentLength <= 0 { + api.WriteErrorResponse(r.Context(), w, api.GetAPIError(api.ErrMissingContentLength), r.URL) + return + } + + // Unmarshal list of keys to be deleted. + requested := &DeleteObjectsRequest{} + if err := xml.NewDecoder(r.Body).Decode(requested); err != nil { + api.WriteErrorResponse(r.Context(), w, err, r.URL) + return + } + + removed := make(map[string]struct{}) + toRemove := make([]string, 0, len(requested.Objects)) + for _, obj := range requested.Objects { + removed[obj.ObjectName] = struct{}{} + toRemove = append(toRemove, obj.ObjectName) + } + + response := &DeleteObjectsResponse{ + Errors: make([]DeleteError, 0, len(toRemove)), + DeletedObjects: make([]ObjectIdentifier, 0, len(toRemove)), + } + + if errs := h.obj.DeleteObjects(r.Context(), bkt, toRemove); errs != nil && !requested.Quiet { + h.log.Error("could not delete objects", + zap.String("request_id", rid), + zap.String("bucket_name", bkt), + zap.Strings("object_name", toRemove), + zap.Errors("errors", errs)) + + for _, e := range errs { + if err, ok := e.(*api.DeleteError); ok { + code := "BadRequest" + desc := err.Error() + + if st, ok := status.FromError(err.Err); ok && st != nil { + desc = st.Message() + code = st.Code().String() + } + + response.Errors = append(response.Errors, DeleteError{ + Code: code, + Message: desc, + Key: err.Object, + }) + + delete(removed, err.Object) + } + } + } + + for key := range removed { + response.DeletedObjects = append(response.DeletedObjects, ObjectIdentifier{ObjectName: key}) + } + + if err := api.EncodeToResponse(w, response); err != nil { + h.log.Error("could not write response", + zap.String("request_id", rid), + zap.String("bucket_name", bkt), + zap.Strings("object_name", toRemove), + zap.Error(err)) + + api.WriteErrorResponse(r.Context(), w, api.Error{ + Code: api.GetAPIError(api.ErrInternalError).Code, + Description: err.Error(), + HTTPStatusCode: http.StatusInternalServerError, + }, r.URL) + } } diff --git a/api/handler/head.go b/api/handler/head.go index 2d30d9c8..4fcfe5fc 100644 --- a/api/handler/head.go +++ b/api/handler/head.go @@ -8,6 +8,8 @@ import ( "github.com/nspcc-dev/neofs-s3-gate/api" "github.com/nspcc-dev/neofs-s3-gate/api/layer" "go.uber.org/zap" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) func (h *handler) HeadObjectHandler(w http.ResponseWriter, r *http.Request) { @@ -45,3 +47,34 @@ func (h *handler) HeadObjectHandler(w http.ResponseWriter, r *http.Request) { w.Header().Set("Last-Modified", inf.Created.Format(http.TimeFormat)) } + +func (h *handler) HeadBucketHandler(w http.ResponseWriter, r *http.Request) { + var ( + req = mux.Vars(r) + bkt = req["bucket"] + rid = api.GetRequestID(r.Context()) + ) + + if _, err := h.obj.GetBucketInfo(r.Context(), bkt); err != nil { + h.log.Error("could not fetch object info", + zap.String("request_id", rid), + zap.String("bucket_name", bkt), + zap.Error(err)) + + code := http.StatusBadRequest + if st, ok := status.FromError(err); ok && st != nil { + switch st.Code() { + case codes.NotFound: + code = http.StatusNotFound + case codes.PermissionDenied: + code = http.StatusForbidden + } + } + + api.WriteResponse(w, code, nil, api.MimeNone) + + return + } + + api.WriteResponse(w, http.StatusOK, nil, api.MimeNone) +} diff --git a/api/handler/list.go b/api/handler/list.go index 78b8b4a0..10c983f5 100644 --- a/api/handler/list.go +++ b/api/handler/list.go @@ -86,11 +86,10 @@ func (h *handler) ListBucketsHandler(w http.ResponseWriter, r *http.Request) { } } -func (h *handler) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) { +func (h *handler) listObjects(w http.ResponseWriter, r *http.Request) (*listObjectsArgs, *layer.ListObjectsInfo, error) { var ( err error arg *listObjectsArgs - res *ListObjectsResponse rid = api.GetRequestID(r.Context()) ) @@ -105,7 +104,7 @@ func (h *handler) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) { HTTPStatusCode: http.StatusBadRequest, }, r.URL) - return + return nil, nil, err } list, err := h.obj.ListObjects(r.Context(), &layer.ListObjectsParams{ @@ -125,10 +124,32 @@ func (h *handler) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) { HTTPStatusCode: http.StatusInternalServerError, }, r.URL) - return + return nil, nil, err } - res = &ListObjectsResponse{ + return arg, list, nil +} + +func (h *handler) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) { + var rid = api.GetRequestID(r.Context()) + if arg, list, err := h.listObjects(w, r); err != nil { + // error already sent to client + return + } else if err := api.EncodeToResponse(w, encodeV1(arg, list)); err != nil { + h.log.Error("something went wrong", + zap.String("request_id", rid), + zap.Error(err)) + + api.WriteErrorResponse(r.Context(), w, api.Error{ + Code: api.GetAPIError(api.ErrInternalError).Code, + Description: err.Error(), + HTTPStatusCode: http.StatusInternalServerError, + }, r.URL) + } +} + +func encodeV1(arg *listObjectsArgs, list *layer.ListObjectsInfo) *ListObjectsResponse { + res := &ListObjectsResponse{ Name: arg.Bucket, EncodingType: arg.Encode, Marker: arg.Marker, @@ -155,13 +176,25 @@ func (h *handler) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) { UserMetadata: obj.Headers, LastModified: obj.Created.Format(time.RFC3339), + Owner: Owner{ + ID: obj.Owner.String(), + DisplayName: obj.Owner.String(), + }, + // ETag: "", - // Owner: Owner{}, // StorageClass: "", }) } - if err := api.EncodeToResponse(w, res); err != nil { + return res +} + +func (h *handler) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) { + var rid = api.GetRequestID(r.Context()) + if arg, list, err := h.listObjects(w, r); err != nil { + // error already sent to client + return + } else if err := api.EncodeToResponse(w, encodeV2(arg, list)); err != nil { h.log.Error("something went wrong", zap.String("request_id", rid), zap.Error(err)) @@ -174,6 +207,48 @@ func (h *handler) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) { } } +func encodeV2(arg *listObjectsArgs, list *layer.ListObjectsInfo) *ListObjectsV2Response { + res := &ListObjectsV2Response{ + Name: arg.Bucket, + EncodingType: arg.Encode, + Prefix: arg.Prefix, + MaxKeys: arg.MaxKeys, + Delimiter: arg.Delimeter, + + IsTruncated: list.IsTruncated, + + ContinuationToken: arg.Marker, + NextContinuationToken: list.NextContinuationToken, + } + + // fill common prefixes + for i := range list.Prefixes { + res.CommonPrefixes = append(res.CommonPrefixes, CommonPrefix{ + Prefix: list.Prefixes[i], + }) + } + + // fill contents + for _, obj := range list.Objects { + res.Contents = append(res.Contents, Object{ + Key: obj.Name, + Size: obj.Size, + UserMetadata: obj.Headers, + LastModified: obj.Created.Format(time.RFC3339), + + Owner: Owner{ + ID: obj.Owner.String(), + DisplayName: obj.Owner.String(), + }, + + // ETag: "", + // StorageClass: "", + }) + } + + return res +} + func parseListObjectArgs(r *http.Request) (*listObjectsArgs, error) { var ( err error diff --git a/api/handler/response.go b/api/handler/response.go index d29d263f..7ce57b99 100644 --- a/api/handler/response.go +++ b/api/handler/response.go @@ -14,6 +14,37 @@ type ListBucketsResponse struct { } // Buckets are nested } +// ListObjectsV2Response - format for list objects response. +type ListObjectsV2Response struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"` + + Name string + Prefix string + StartAfter string `xml:"StartAfter,omitempty"` + // When response is truncated (the IsTruncated element value in the response + // is true), you can use the key name in this field as marker in the subsequent + // request to get next set of objects. Server lists objects in alphabetical + // order Note: This element is returned only if you have delimiter request parameter + // specified. If response does not include the NextMaker and it is truncated, + // you can use the value of the last Key in the response as the marker in the + // subsequent request to get the next set of object keys. + ContinuationToken string `xml:"ContinuationToken,omitempty"` + NextContinuationToken string `xml:"NextContinuationToken,omitempty"` + + KeyCount int + MaxKeys int + Delimiter string + // A flag that indicates whether or not ListObjects returned all of the results + // that satisfied the search criteria. + IsTruncated bool + + Contents []Object + CommonPrefixes []CommonPrefix + + // Encoding type used to encode object keys in the response. + EncodingType string `xml:"EncodingType,omitempty"` +} + // Bucket container for bucket metadata type Bucket struct { Name string diff --git a/api/handler/unimplemented.go b/api/handler/unimplemented.go index 378efc05..2525549d 100644 --- a/api/handler/unimplemented.go +++ b/api/handler/unimplemented.go @@ -295,14 +295,6 @@ func (h *handler) ListObjectsV2MHandler(w http.ResponseWriter, r *http.Request) }, r.URL) } -func (h *handler) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) { - api.WriteErrorResponse(r.Context(), w, api.Error{ - Code: "XNeoFSUnimplemented", - Description: "implement me " + mux.CurrentRoute(r).GetName(), - HTTPStatusCode: http.StatusNotImplemented, - }, r.URL) -} - func (h *handler) ListBucketObjectVersionsHandler(w http.ResponseWriter, r *http.Request) { api.WriteErrorResponse(r.Context(), w, api.Error{ Code: "XNeoFSUnimplemented", @@ -343,14 +335,6 @@ func (h *handler) PutBucketObjectLockConfigHandler(w http.ResponseWriter, r *htt }, r.URL) } -func (h *handler) HeadBucketHandler(w http.ResponseWriter, r *http.Request) { - api.WriteErrorResponse(r.Context(), w, api.Error{ - Code: "XNeoFSUnimplemented", - Description: "implement me " + mux.CurrentRoute(r).GetName(), - HTTPStatusCode: http.StatusNotImplemented, - }, r.URL) -} - func (h *handler) PostPolicyBucketHandler(w http.ResponseWriter, r *http.Request) { api.WriteErrorResponse(r.Context(), w, api.Error{ Code: "XNeoFSUnimplemented", diff --git a/api/headers.go b/api/headers.go new file mode 100644 index 00000000..225e24f2 --- /dev/null +++ b/api/headers.go @@ -0,0 +1,25 @@ +package api + +// Standard S3 HTTP response constants +const ( + LastModified = "Last-Modified" + Date = "Date" + ETag = "ETag" + ContentType = "Content-Type" + ContentMD5 = "Content-Md5" + ContentEncoding = "Content-Encoding" + Expires = "Expires" + ContentLength = "Content-Length" + ContentLanguage = "Content-Language" + ContentRange = "Content-Range" + Connection = "Connection" + AcceptRanges = "Accept-Ranges" + AmzBucketRegion = "X-Amz-Bucket-Region" + ServerInfo = "Server" + RetryAfter = "Retry-After" + Location = "Location" + CacheControl = "Cache-Control" + ContentDisposition = "Content-Disposition" + Authorization = "Authorization" + Action = "Action" +) diff --git a/api/layer/layer.go b/api/layer/layer.go index 2dc95b08..0cb2c702 100644 --- a/api/layer/layer.go +++ b/api/layer/layer.go @@ -12,8 +12,9 @@ import ( "github.com/nspcc-dev/neofs-api-go/service" "github.com/nspcc-dev/neofs-s3-gate/api" "github.com/nspcc-dev/neofs-s3-gate/api/pool" - "github.com/pkg/errors" "go.uber.org/zap" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) type ( @@ -68,7 +69,7 @@ type ( ListObjects(ctx context.Context, p *ListObjectsParams) (*ListObjectsInfo, error) DeleteObject(ctx context.Context, bucket, object string) error - DeleteObjects(ctx context.Context, bucket string, objects []string) ([]error, error) + DeleteObjects(ctx context.Context, bucket string, objects []string) []error } ) @@ -142,7 +143,7 @@ func (n *layer) GetBucketInfo(ctx context.Context, name string) (*BucketInfo, er } } - return nil, errors.New("bucket not found") + return nil, status.Error(codes.NotFound, "bucket not found") } // ListBuckets returns all user containers. Name of the bucket is a container @@ -224,6 +225,7 @@ func (n *layer) ListObjects(ctx context.Context, p *ListObjectsParams) (*ListObj oi = objectInfoFromMeta(meta) } else { // if there are sub-entities in tail - dir oi = &ObjectInfo{ + Owner: meta.SystemHeader.OwnerID, Bucket: meta.SystemHeader.CID.String(), Name: tail[:ind+1], // dir MUST have slash symbol in the end // IsDir: true, @@ -375,17 +377,26 @@ func (n *layer) CopyObject(ctx context.Context, p *CopyObjectParams) (*ObjectInf func (n *layer) DeleteObject(ctx context.Context, bucket, object string) error { cid, err := refs.CIDFromString(bucket) if err != nil { - return err + return &api.DeleteError{ + Err: err, + Object: object, + } } ids, err := n.objectFindIDs(ctx, cid, object) if err != nil { - return errors.Wrap(err, "could not find object") + return &api.DeleteError{ + Err: err, + Object: object, + } } for _, id := range ids { if err = n.objectDelete(ctx, delParams{addr: refs.Address{CID: cid, ObjectID: id}}); err != nil { - return errors.Wrapf(err, "could not remove object: %s => %s", object, id) + return &api.DeleteError{ + Err: err, + Object: object, + } } } @@ -393,12 +404,14 @@ func (n *layer) DeleteObject(ctx context.Context, bucket, object string) error { } // DeleteObjects from the storage. -func (n *layer) DeleteObjects(ctx context.Context, bucket string, objects []string) ([]error, error) { +func (n *layer) DeleteObjects(ctx context.Context, bucket string, objects []string) []error { var errs = make([]error, 0, len(objects)) for i := range objects { - errs = append(errs, n.DeleteObject(ctx, bucket, objects[i])) + if err := n.DeleteObject(ctx, bucket, objects[i]); err != nil { + errs = append(errs, err) + } } - return errs, nil + return errs } diff --git a/api/layer/object.go b/api/layer/object.go index c92edaff..a8906fac 100644 --- a/api/layer/object.go +++ b/api/layer/object.go @@ -13,6 +13,8 @@ import ( "github.com/nspcc-dev/neofs-s3-gate/api/pool" "github.com/pkg/errors" "go.uber.org/zap" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) const ( @@ -226,7 +228,7 @@ func (n *layer) objectFindID(ctx context.Context, cid refs.CID, name string, put return id, nil } } - return id, errors.New("object not found") + return id, status.Error(codes.NotFound, "object not found") } else if ln == 1 { return result[0], nil } diff --git a/api/layer/util.go b/api/layer/util.go index 91acd698..d97f13ed 100644 --- a/api/layer/util.go +++ b/api/layer/util.go @@ -7,6 +7,7 @@ import ( "time" "github.com/nspcc-dev/neofs-api-go/object" + "github.com/nspcc-dev/neofs-api-go/refs" ) type ( @@ -16,6 +17,7 @@ type ( Size int64 ContentType string Created time.Time + Owner refs.OwnerID Headers map[string]string } diff --git a/api/response.go b/api/response.go index 60029582..b53c2f58 100644 --- a/api/response.go +++ b/api/response.go @@ -118,21 +118,25 @@ var s3ErrorResponseMap = map[string]string{ } // WriteErrorResponse writes error headers -func WriteErrorResponse(ctx context.Context, w http.ResponseWriter, err Error, reqURL *url.URL) { - switch err.Code { - case "SlowDown", "XNeoFSServerNotInitialized", "XNeoFSReadQuorum", "XNeoFSWriteQuorum": - // Set retry-after header to indicate user-agents to retry request after 120secs. - // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After - w.Header().Set(hdrRetryAfter, "120") - case "AccessDenied": - // TODO process when the request is from browser and also if browser +func WriteErrorResponse(ctx context.Context, w http.ResponseWriter, err error, reqURL *url.URL) { + code := http.StatusBadRequest + + if e, ok := err.(Error); ok { + switch e.Code { + case "SlowDown", "XNeoFSServerNotInitialized", "XNeoFSReadQuorum", "XNeoFSWriteQuorum": + // Set retry-after header to indicate user-agents to retry request after 120secs. + // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After + w.Header().Set(hdrRetryAfter, "120") + case "AccessDenied": + // TODO process when the request is from browser and also if browser + } } // Generate error response. errorResponse := getAPIErrorResponse(ctx, err, reqURL.Path, w.Header().Get(hdrAmzRequestID), deploymentID.String()) encodedErrorResponse := EncodeResponse(errorResponse) - writeResponse(w, err.HTTPStatusCode, encodedErrorResponse, mimeXML) + WriteResponse(w, code, encodedErrorResponse, MimeXML) } // If none of the http routes match respond with appropriate errors @@ -162,9 +166,9 @@ func removeSensitiveHeaders(h http.Header) { h.Del(hdrSSECopyKey) } -func writeResponse(w http.ResponseWriter, statusCode int, response []byte, mType mimeType) { +func WriteResponse(w http.ResponseWriter, statusCode int, response []byte, mType mimeType) { setCommonHeaders(w) - if mType != mimeNone { + if mType != MimeNone { w.Header().Set(hdrContentType, string(mType)) } w.Header().Set(hdrContentLength, strconv.Itoa(len(response))) @@ -196,11 +200,11 @@ func EncodeToResponse(w http.ResponseWriter, response interface{}) error { // WriteSuccessResponseXML writes success headers and response if any, // with content-type set to `application/xml`. func WriteSuccessResponseXML(w http.ResponseWriter, response []byte) { - writeResponse(w, http.StatusOK, response, mimeXML) + WriteResponse(w, http.StatusOK, response, MimeXML) } func WriteSuccessResponseHeadersOnly(w http.ResponseWriter) { - writeResponse(w, http.StatusOK, nil, mimeNone) + WriteResponse(w, http.StatusOK, nil, MimeNone) } // Error - Returns S3 error string. diff --git a/api/router.go b/api/router.go index c947ac5d..3cceeef9 100644 --- a/api/router.go +++ b/api/router.go @@ -90,12 +90,11 @@ const ( // SlashSeparator - slash separator. SlashSeparator = "/" - // Means no response type. - mimeNone mimeType = "" - // Means response type is JSON. - // mimeJSON mimeType = "application/json" - // Means response type is XML. - mimeXML mimeType = "application/xml" + // MimeNone means no response type. + MimeNone mimeType = "" + + // MimeXML means response type is XML. + MimeXML mimeType = "application/xml" ) var _ = logErrorResponse