dkirillov/frostfs-s3-gw
1
0
Fork 0
forked from TrueCloudLab/frostfs-s3-gw
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge pull request #22 from nspcc-dev/api-handlers

Browse source 
API Handlers
This commit is contained in:
Evgeniy Kulikov 2020-08-20 14:07:52 +03:00 committed by GitHub
commit a3c95cffb1
13 changed files with 480 additions and 121 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

96
api/handler/copy.go Normal file
View file

@ -0,0 +1,96 @@
package handler
import (
"net/http"
"net/url"
"strings"
"time"
"github.com/gorilla/mux"
"github.com/nspcc-dev/neofs-s3-gate/api"
"github.com/nspcc-dev/neofs-s3-gate/api/layer"
"go.uber.org/zap"
)
// path2BucketObject returns bucket and object.
func path2BucketObject(path string) (bucket, prefix string) {
path = strings.TrimPrefix(path, api.SlashSeparator)
m := strings.Index(path, api.SlashSeparator)
if m < 0 {
return path, ""
}
return path[:m], path[m+len(api.SlashSeparator):]
}
func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
var (
err error
inf *layer.ObjectInfo
req = mux.Vars(r)
bkt = req["bucket"]
obj = req["object"]
rid = api.GetRequestID(r.Context())
)
src := r.Header.Get("X-Amz-Copy-Source")
// Check https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectVersioning.html
// Regardless of whether you have enabled versioning, each object in your bucket
// has a version ID. If you have not enabled versioning, Amazon S3 sets the value
// of the version ID to null. If you have enabled versioning, Amazon S3 assigns a
// unique version ID value for the object.
if u, err := url.Parse(src); err == nil {
// Check if versionId query param was added, if yes then check if
// its non "null" value, we should error out since we do not support
// any versions other than "null".
if vid := u.Query().Get("versionId"); vid != "" && vid != "null" {
api.WriteErrorResponse(r.Context(), w, api.Error{
Code: api.GetAPIError(api.ErrNoSuchVersion).Code,
Description: "",
HTTPStatusCode: http.StatusBadRequest,
}, r.URL)
return
}
src = u.Path
}
srcBucket, srcObject := path2BucketObject(src)
params := &layer.CopyObjectParams{
SrcBucket: srcBucket,
DstBucket: bkt,
SrcObject: srcObject,
DstObject: obj,
}
if inf, err = h.obj.CopyObject(r.Context(), params); err != nil {
h.log.Error("could not copy object",
zap.String("request_id", rid),
zap.String("dst_bucket_name", bkt),
zap.String("dst_object_name", obj),
zap.String("src_bucket_name", srcBucket),
zap.String("src_object_name", srcObject),
zap.Error(err))
api.WriteErrorResponse(r.Context(), w, api.Error{
Code: api.GetAPIError(api.ErrInternalError).Code,
Description: err.Error(),
HTTPStatusCode: http.StatusInternalServerError,
}, r.URL)
} else if err = api.EncodeToResponse(w, &CopyObjectResponse{LastModified: inf.Created.Format(time.RFC3339)}); err != nil {
h.log.Error("something went wrong",
zap.String("request_id", rid),
zap.String("dst_bucket_name", bkt),
zap.String("dst_object_name", obj),
zap.String("src_bucket_name", srcBucket),
zap.String("src_object_name", srcObject),
zap.Error(err))
api.WriteErrorResponse(r.Context(), w, api.Error{
Code: api.GetAPIError(api.ErrInternalError).Code,
Description: err.Error(),
HTTPStatusCode: http.StatusInternalServerError,
}, r.URL)
}
}

36
api/handler/delete.go Normal file
View file

@ -0,0 +1,36 @@
package handler
import (
"net/http"
"github.com/gorilla/mux"
"github.com/nspcc-dev/neofs-s3-gate/api"
"go.uber.org/zap"
)
func (h *handler) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) {
var (
req = mux.Vars(r)
bkt = req["bucket"]
obj = req["object"]
rid = api.GetRequestID(r.Context())
)
if err := h.obj.DeleteObject(r.Context(), bkt, obj); err != nil {
h.log.Error("could not delete object",
zap.String("request_id", rid),
zap.String("bucket_name", bkt),
zap.String("object_name", obj),
zap.Error(err))
// Ignore delete errors:
// api.WriteErrorResponse(r.Context(), w, api.Error{
// Code: api.GetAPIError(api.ErrInternalError).Code,
// Description: err.Error(),
// HTTPStatusCode: http.StatusInternalServerError,
// }, r.URL)
}
w.WriteHeader(http.StatusNoContent)
}

69
api/handler/get.go Normal file
View file

@ -0,0 +1,69 @@
package handler
import (
"net/http"
"strconv"
"github.com/gorilla/mux"
"github.com/nspcc-dev/neofs-s3-gate/api"
"github.com/nspcc-dev/neofs-s3-gate/api/layer"
"go.uber.org/zap"
)
func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
var (
err error
inf *layer.ObjectInfo
req = mux.Vars(r)
bkt = req["bucket"]
obj = req["object"]
rid = api.GetRequestID(r.Context())
)
params := &layer.GetObjectParams{
Bucket: bkt,
Object: obj,
Writer: w,
}
if inf, err = h.obj.GetObjectInfo(r.Context(), bkt, obj); err != nil {
h.log.Error("could not find object",
zap.String("request_id", rid),
zap.String("bucket_name", bkt),
zap.String("object_name", obj),
zap.Error(err))
api.WriteErrorResponse(r.Context(), w, api.Error{
Code: api.GetAPIError(api.ErrInternalError).Code,
Description: err.Error(),
HTTPStatusCode: http.StatusInternalServerError,
}, r.URL)
return
}
params.Length = inf.Size
if err = h.obj.GetObject(r.Context(), params); err != nil {
h.log.Error("could not get object",
zap.String("request_id", rid),
zap.String("bucket_name", bkt),
zap.String("object_name", obj),
zap.Error(err))
api.WriteErrorResponse(r.Context(), w, api.Error{
Code: api.GetAPIError(api.ErrInternalError).Code,
Description: err.Error(),
HTTPStatusCode: http.StatusInternalServerError,
}, r.URL)
return
}
w.Header().Set("Content-Type", inf.ContentType)
w.Header().Set("Content-Length", strconv.FormatInt(inf.Size, 10))
w.Header().Set("Last-Modified", inf.Created.Format(http.TimeFormat))
}

47
api/handler/head.go Normal file
View file

@ -0,0 +1,47 @@
package handler
import (
"net/http"
"strconv"
"github.com/gorilla/mux"
"github.com/nspcc-dev/neofs-s3-gate/api"
"github.com/nspcc-dev/neofs-s3-gate/api/layer"
"go.uber.org/zap"
)
func (h *handler) HeadObjectHandler(w http.ResponseWriter, r *http.Request) {
var (
err error
inf *layer.ObjectInfo
req = mux.Vars(r)
bkt = req["bucket"]
obj = req["object"]
rid = api.GetRequestID(r.Context())
)
if inf, err = h.obj.GetObjectInfo(r.Context(), bkt, obj); err != nil {
h.log.Error("could not fetch object info",
zap.String("request_id", rid),
zap.String("bucket_name", bkt),
zap.String("object_name", obj),
zap.Error(err))
api.WriteErrorResponse(r.Context(), w, api.Error{
Code: api.GetAPIError(api.ErrInternalError).Code,
Description: err.Error(),
HTTPStatusCode: http.StatusInternalServerError,
}, r.URL)
return
}
w.WriteHeader(http.StatusOK)
w.Header().Set("Content-Type", inf.ContentType)
w.Header().Set("Content-Length", strconv.FormatInt(inf.Size, 10))
w.Header().Set("Last-Modified", inf.Created.Format(http.TimeFormat))
}

73
api/handler/list-buckets.go
View file

@ -1,73 +0,0 @@
package handler
import (
"net/http"
"time"
"github.com/nspcc-dev/neofs-s3-gate/api"
"github.com/nspcc-dev/neofs-s3-gate/auth"
"go.uber.org/zap"
)
func (h *handler) ListBucketsHandler(w http.ResponseWriter, r *http.Request) {
var (
res *ListBucketsResponse
rid = api.GetRequestID(r.Context())
)
tkn, err := auth.GetBearerToken(r.Context())
if err != nil {
h.log.Error("something went wrong",
zap.String("request_id", rid),
zap.Error(err))
api.WriteErrorResponse(r.Context(), w, api.Error{
Code: api.GetAPIError(api.ErrInternalError).Code,
Description: err.Error(),
HTTPStatusCode: http.StatusInternalServerError,
}, r.URL)
return
}
list, err := h.obj.ListBuckets(r.Context())
if err != nil {
h.log.Error("something went wrong",
zap.String("request_id", rid),
zap.Error(err))
api.WriteErrorResponse(r.Context(), w, api.Error{
Code: api.GetAPIError(api.ErrInternalError).Code,
Description: err.Error(),
HTTPStatusCode: http.StatusInternalServerError,
}, r.URL)
return
}
res = &ListBucketsResponse{
Owner: Owner{
ID: tkn.OwnerID.String(),
DisplayName: tkn.OwnerID.String(),
},
}
for _, item := range list {
res.Buckets.Buckets = append(res.Buckets.Buckets, Bucket{
Name: item.Name,
CreationDate: item.Created.Format(time.RFC3339),
})
}
if err = api.EncodeToResponse(w, res); err != nil {
h.log.Error("something went wrong",
zap.String("request_id", rid),
zap.Error(err))
api.WriteErrorResponse(r.Context(), w, api.Error{
Code: api.GetAPIError(api.ErrInternalError).Code,
Description: err.Error(),
HTTPStatusCode: http.StatusInternalServerError,
}, r.URL)
}
}

71
api/handler/list-objects.go → api/handler/list.go
View file

@ -5,9 +5,9 @@ import (
"strconv"
"time"
"github.com/gorilla/mux"
"github.com/nspcc-dev/neofs-s3-gate/api"
"github.com/nspcc-dev/neofs-s3-gate/api/layer"
"github.com/nspcc-dev/neofs-s3-gate/auth"
"go.uber.org/zap"
)
@ -23,6 +23,69 @@ type listObjectsArgs struct {
var maxObjectList = 10000 // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse.
func (h *handler) ListBucketsHandler(w http.ResponseWriter, r *http.Request) {
var (
res *ListBucketsResponse
rid = api.GetRequestID(r.Context())
)
tkn, err := auth.GetBearerToken(r.Context())
if err != nil {
h.log.Error("something went wrong",
zap.String("request_id", rid),
zap.Error(err))
api.WriteErrorResponse(r.Context(), w, api.Error{
Code: api.GetAPIError(api.ErrInternalError).Code,
Description: err.Error(),
HTTPStatusCode: http.StatusInternalServerError,
}, r.URL)
return
}
list, err := h.obj.ListBuckets(r.Context())
if err != nil {
h.log.Error("something went wrong",
zap.String("request_id", rid),
zap.Error(err))
api.WriteErrorResponse(r.Context(), w, api.Error{
Code: api.GetAPIError(api.ErrInternalError).Code,
Description: err.Error(),
HTTPStatusCode: http.StatusInternalServerError,
}, r.URL)
return
}
res = &ListBucketsResponse{
Owner: Owner{
ID: tkn.OwnerID.String(),
DisplayName: tkn.OwnerID.String(),
},
}
for _, item := range list {
res.Buckets.Buckets = append(res.Buckets.Buckets, Bucket{
Name: item.Name,
CreationDate: item.Created.Format(time.RFC3339),
})
}
if err = api.EncodeToResponse(w, res); err != nil {
h.log.Error("something went wrong",
zap.String("request_id", rid),
zap.Error(err))
api.WriteErrorResponse(r.Context(), w, api.Error{
Code: api.GetAPIError(api.ErrInternalError).Code,
Description: err.Error(),
HTTPStatusCode: http.StatusInternalServerError,
}, r.URL)
}
}
func (h *handler) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {
var (
err error
@ -104,9 +167,9 @@ func (h *handler) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {
zap.Error(err))
api.WriteErrorResponse(r.Context(), w, api.Error{
Code: "XNeoFSUnimplemented",
Description: "implement me " + mux.CurrentRoute(r).GetName(),
HTTPStatusCode: http.StatusNotImplemented,
Code: api.GetAPIError(api.ErrInternalError).Code,
Description: err.Error(),
HTTPStatusCode: http.StatusInternalServerError,
}, r.URL)
}
}

74
api/handler/put.go Normal file
View file

@ -0,0 +1,74 @@
package handler
import (
"net/http"
"github.com/gorilla/mux"
"github.com/nspcc-dev/neofs-s3-gate/api"
"github.com/nspcc-dev/neofs-s3-gate/api/layer"
"go.uber.org/zap"
)
func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
var (
err error
req = mux.Vars(r)
bkt = req["bucket"]
obj = req["object"]
rid = api.GetRequestID(r.Context())
)
if _, err := h.obj.GetBucketInfo(r.Context(), bkt); err != nil {
h.log.Error("could not find bucket",
zap.String("request_id", rid),
zap.String("bucket_name", bkt),
zap.Error(err))
api.WriteErrorResponse(r.Context(), w, api.Error{
Code: api.GetAPIError(api.ErrBadRequest).Code,
Description: err.Error(),
HTTPStatusCode: http.StatusBadRequest,
}, r.URL)
return
} else if _, err = h.obj.GetObjectInfo(r.Context(), bkt, obj); err == nil {
h.log.Error("object exists",
zap.String("request_id", rid),
zap.String("bucket_name", bkt),
zap.String("object_name", obj),
zap.Error(err))
api.WriteErrorResponse(r.Context(), w, api.Error{
Code: api.GetAPIError(api.ErrMethodNotAllowed).Code,
Description: "Object: " + bkt + "#" + obj + " already exists",
HTTPStatusCode: http.StatusBadRequest,
}, r.URL)
return
}
params := &layer.PutObjectParams{
Bucket: bkt,
Object: obj,
Reader: r.Body,
Size: r.ContentLength,
}
if _, err = h.obj.PutObject(r.Context(), params); err != nil {
h.log.Error("could not upload object",
zap.String("request_id", rid),
zap.String("bucket_name", bkt),
zap.String("object_name", obj),
zap.Error(err))
api.WriteErrorResponse(r.Context(), w, api.Error{
Code: api.GetAPIError(api.ErrInternalError).Code,
Description: err.Error(),
HTTPStatusCode: http.StatusInternalServerError,
}, r.URL)
return
}
api.WriteSuccessResponseHeadersOnly(w)
}

35
api/handler/response.go
View file

@ -86,3 +86,38 @@ type LocationResponse struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LocationConstraint" json:"-"`
Location string `xml:",chardata"`
}
// CopyObjectResponse container returns ETag and LastModified of the successfully copied object
type CopyObjectResponse struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopyObjectResult" json:"-"`
LastModified string // time string of format "2006-01-02T15:04:05.000Z"
ETag string // md5sum of the copied object.
}
// MarshalXML - StringMap marshals into XML.
func (s StringMap) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
tokens := []xml.Token{start}
for key, value := range s {
t := xml.StartElement{}
t.Name = xml.Name{
Space: "",
Local: key,
}
tokens = append(tokens, t, xml.CharData(value), xml.EndElement{Name: t.Name})
}
tokens = append(tokens, xml.EndElement{
Name: start.Name,
})
for _, t := range tokens {
if err := e.EncodeToken(t); err != nil {
return err
}
}
// flush to ensure tokens are written
return e.Flush()
}

40
api/handler/unimplemented.go
View file

@ -7,14 +7,6 @@ import (
"github.com/nspcc-dev/neofs-s3-gate/api"
)
func (h *handler) HeadObjectHandler(w http.ResponseWriter, r *http.Request) {
api.WriteErrorResponse(r.Context(), w, api.Error{
Code: "XNeoFSUnimplemented",
Description: "implement me " + mux.CurrentRoute(r).GetName(),
HTTPStatusCode: http.StatusNotImplemented,
}, r.URL)
}
func (h *handler) CopyObjectPartHandler(w http.ResponseWriter, r *http.Request) {
api.WriteErrorResponse(r.Context(), w, api.Error{
Code: "XNeoFSUnimplemented",
@ -127,22 +119,6 @@ func (h *handler) GetObjectLegalHoldHandler(w http.ResponseWriter, r *http.Reque
}, r.URL)
}
func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
api.WriteErrorResponse(r.Context(), w, api.Error{
Code: "XNeoFSUnimplemented",
Description: "implement me " + mux.CurrentRoute(r).GetName(),
HTTPStatusCode: http.StatusNotImplemented,
}, r.URL)
}
func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
api.WriteErrorResponse(r.Context(), w, api.Error{
Code: "XNeoFSUnimplemented",
Description: "implement me " + mux.CurrentRoute(r).GetName(),
HTTPStatusCode: http.StatusNotImplemented,
}, r.URL)
}
func (h *handler) PutObjectRetentionHandler(w http.ResponseWriter, r *http.Request) {
api.WriteErrorResponse(r.Context(), w, api.Error{
Code: "XNeoFSUnimplemented",
@ -159,22 +135,6 @@ func (h *handler) PutObjectLegalHoldHandler(w http.ResponseWriter, r *http.Reque
}, r.URL)
}
func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
api.WriteErrorResponse(r.Context(), w, api.Error{
Code: "XNeoFSUnimplemented",
Description: "implement me " + mux.CurrentRoute(r).GetName(),
HTTPStatusCode: http.StatusNotImplemented,
}, r.URL)
}
func (h *handler) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) {
api.WriteErrorResponse(r.Context(), w, api.Error{
Code: "XNeoFSUnimplemented",
Description: "implement me " + mux.CurrentRoute(r).GetName(),
HTTPStatusCode: http.StatusNotImplemented,
}, r.URL)
}
func (h *handler) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
api.WriteErrorResponse(r.Context(), w, api.Error{
Code: "XNeoFSUnimplemented",

12
api/layer/layer.go
View file

@ -10,6 +10,7 @@ import (
"github.com/nspcc-dev/neofs-api-go/object"
"github.com/nspcc-dev/neofs-api-go/refs"
"github.com/nspcc-dev/neofs-api-go/service"
"github.com/nspcc-dev/neofs-s3-gate/api"
"github.com/nspcc-dev/neofs-s3-gate/api/pool"
"github.com/pkg/errors"
"go.uber.org/zap"
@ -44,6 +45,7 @@ type (
DstBucket string
SrcObject string
DstObject string
Header map[string]string
}
NeoFS interface {
@ -288,7 +290,10 @@ func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*ObjectInfo,
_, err = n.objectFindID(ctx, cid, p.Object, true)
if err == nil {
return nil, err
return nil, &api.ObjectAlreadyExists{
Bucket: p.Bucket,
Object: p.Object,
}
}
oid, err := refs.NewObjectID()
@ -352,6 +357,11 @@ func (n *layer) CopyObject(ctx context.Context, p *CopyObjectParams) (*ObjectInf
_ = pw.CloseWithError(err)
}()
// set custom headers
for k, v := range p.Header {
info.Headers[k] = v
}
return n.PutObject(ctx, &PutObjectParams{
Bucket: p.DstBucket,
Object: p.DstObject,

3
api/layer/object.go
View file

@ -390,7 +390,10 @@ func (n *layer) objectPut(ctx context.Context, p putParams) (*object.Object, err
p.userHeaders = make(map[string]string)
}
// Set object name if not set before
if _, ok := p.userHeaders[AWS3NameHeader]; !ok {
p.userHeaders[AWS3NameHeader] = p.name
}
readBuffer := make([]byte, dataChunkSize)
obj := &object.Object{

4
api/response.go
View file

@ -199,6 +199,10 @@ func WriteSuccessResponseXML(w http.ResponseWriter, response []byte) {
writeResponse(w, http.StatusOK, response, mimeXML)
}
func WriteSuccessResponseHeadersOnly(w http.ResponseWriter) {
writeResponse(w, http.StatusOK, nil, mimeNone)
}
// Error - Returns S3 error string.
func (e ErrorResponse) Error() string {
if e.Message == "" {

39
api/router.go
View file

@ -79,6 +79,11 @@ type (
// mimeType represents various MIME type used API responses.
mimeType string
logResponseWriter struct {
http.ResponseWriter
statusCode int
}
)
const (
@ -93,6 +98,13 @@ const (
mimeXML mimeType = "application/xml"
)
var _ = logErrorResponse
func (lrw *logResponseWriter) WriteHeader(code int) {
lrw.statusCode = code
lrw.ResponseWriter.WriteHeader(code)
}
func setRequestID(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// generate random UUIDv4
@ -114,6 +126,24 @@ func setRequestID(h http.Handler) http.Handler {
})
}
func logErrorResponse(l *zap.Logger) mux.MiddlewareFunc {
return func(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
lw := &logResponseWriter{ResponseWriter: w}
// pass execution:
h.ServeHTTP(lw, r)
// Ignore <300 status codes
if lw.statusCode >= http.StatusMultipleChoices {
l.Error("something went wrong",
zap.Int("status", lw.statusCode),
zap.String("method", mux.CurrentRoute(r).GetName()))
}
})
}
}
func GetRequestID(v interface{}) string {
switch t := v.(type) {
case context.Context:
@ -128,8 +158,13 @@ func GetRequestID(v interface{}) string {
func Attach(r *mux.Router, m MaxClients, h Handler, center *auth.Center, log *zap.Logger) {
api := r.PathPrefix(SlashSeparator).Subrouter()
// Attach behaviors: RequestID, ...
api.Use(setRequestID)
api.Use(
// -- prepare request
setRequestID,
// -- logging error requests
// logErrorResponse(log),
)
// Attach user authentication for all S3 routes.
AttachUserAuth(api, center, log)