TrueCloudLab/frostfs-s3-gw
17
3
Fork 19
Code Issues 30 Pull requests 9 Projects Releases 18 Packages Activity Actions

Remove legacy code usage

Browse source
This commit is contained in:
Evgeniy Kulikov 2020-08-06 13:49:25 +03:00
parent 28fa75fb69
commit e223876d53
5 changed files with 135 additions and 355 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

129
neofs/api/errors.go
View file

@ -8,17 +8,9 @@ import (
"net/url"
"strings"
"github.com/Azure/azure-storage-blob-go/azblob"
"github.com/minio/minio-go/v6"
"github.com/minio/minio-go/v6/pkg/tags"
"github.com/minio/minio-go/v7/pkg/tags"
"github.com/minio/minio/auth"
"github.com/minio/minio/neofs/api/crypto"
"github.com/minio/minio/pkg/bucket/lifecycle"
objectlock "github.com/minio/minio/pkg/bucket/object/lock"
"github.com/minio/minio/pkg/bucket/policy"
"github.com/minio/minio/pkg/event"
"github.com/minio/minio/pkg/hash"
"google.golang.org/api/googleapi"
)
type (
@ -1629,7 +1621,7 @@ func (e errorCodeMap) ToAPIErr(errCode ErrorCode) Error {
// toAPIErrorCode - Converts embedded errors. Convenience
// function written to handle all cases where we have known types of
// errors returned by underlying layers.
func toAPIErrorCode(ctx context.Context, err error) (apiErr ErrorCode) {
func toAPIErrorCode(_ context.Context, err error) (apiErr ErrorCode) {
if err == nil {
return ErrNone
}
@ -1690,16 +1682,16 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr ErrorCode) {
apiErr = ErrOperationTimedOut
case errDiskNotFound:
apiErr = ErrSlowDown
case objectlock.ErrInvalidRetentionDate:
apiErr = ErrInvalidRetentionDate
case objectlock.ErrPastObjectLockRetainDate:
apiErr = ErrPastObjectLockRetainDate
case objectlock.ErrUnknownWORMModeDirective:
apiErr = ErrUnknownWORMModeDirective
case objectlock.ErrObjectLockInvalidHeaders:
apiErr = ErrObjectLockInvalidHeaders
case objectlock.ErrMalformedXML:
apiErr = ErrMalformedXML
// case objectlock.ErrInvalidRetentionDate:
// apiErr = ErrInvalidRetentionDate
// case objectlock.ErrPastObjectLockRetainDate:
// apiErr = ErrPastObjectLockRetainDate
// case objectlock.ErrUnknownWORMModeDirective:
// apiErr = ErrUnknownWORMModeDirective
// case objectlock.ErrObjectLockInvalidHeaders:
// apiErr = ErrObjectLockInvalidHeaders
// case objectlock.ErrMalformedXML:
// apiErr = ErrMalformedXML
}
// Compression errors
@ -1722,8 +1714,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr ErrorCode) {
switch err.(type) {
case StorageFull:
apiErr = ErrStorageFull
case hash.BadDigest:
apiErr = ErrBadDigest
// case hash.BadDigest:
// apiErr = ErrBadDigest
case AllAccessDisabled:
apiErr = ErrAllAccessDisabled
case IncompleteBody:
@ -1774,8 +1766,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr ErrorCode) {
apiErr = ErrEntityTooSmall
case SignatureDoesNotMatch:
apiErr = ErrSignatureDoesNotMatch
case hash.SHA256Mismatch:
apiErr = ErrContentSHA256Mismatch
// case hash.SHA256Mismatch:
// apiErr = ErrContentSHA256Mismatch
case ObjectTooLarge:
apiErr = ErrEntityTooLarge
case ObjectTooSmall:
@ -1800,28 +1792,28 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr ErrorCode) {
apiErr = ErrAdminNoSuchQuotaConfiguration
case BucketQuotaExceeded:
apiErr = ErrAdminBucketQuotaExceeded
case *event.ErrInvalidEventName:
apiErr = ErrEventNotification
case *event.ErrInvalidARN:
apiErr = ErrARNNotification
case *event.ErrARNNotFound:
apiErr = ErrARNNotification
case *event.ErrUnknownRegion:
apiErr = ErrRegionNotification
case *event.ErrInvalidFilterName:
apiErr = ErrFilterNameInvalid
case *event.ErrFilterNamePrefix:
apiErr = ErrFilterNamePrefix
case *event.ErrFilterNameSuffix:
apiErr = ErrFilterNameSuffix
case *event.ErrInvalidFilterValue:
apiErr = ErrFilterValueInvalid
case *event.ErrDuplicateEventName:
apiErr = ErrOverlappingConfigs
case *event.ErrDuplicateQueueConfiguration:
apiErr = ErrOverlappingFilterNotification
case *event.ErrUnsupportedConfiguration:
apiErr = ErrUnsupportedNotification
// case *event.ErrInvalidEventName:
// apiErr = ErrEventNotification
// case *event.ErrInvalidARN:
// apiErr = ErrARNNotification
// case *event.ErrARNNotFound:
// apiErr = ErrARNNotification
// case *event.ErrUnknownRegion:
// apiErr = ErrRegionNotification
// case *event.ErrInvalidFilterName:
// apiErr = ErrFilterNameInvalid
// case *event.ErrFilterNamePrefix:
// apiErr = ErrFilterNamePrefix
// case *event.ErrFilterNameSuffix:
// apiErr = ErrFilterNameSuffix
// case *event.ErrInvalidFilterValue:
// apiErr = ErrFilterValueInvalid
// case *event.ErrDuplicateEventName:
// apiErr = ErrOverlappingConfigs
// case *event.ErrDuplicateQueueConfiguration:
// apiErr = ErrOverlappingFilterNotification
// case *event.ErrUnsupportedConfiguration:
// apiErr = ErrUnsupportedNotification
case OperationTimedOut:
apiErr = ErrOperationTimedOut
case BackendDown:
@ -1884,55 +1876,36 @@ func toAPIError(ctx context.Context, err error) Error {
e.Error()),
HTTPStatusCode: http.StatusBadRequest,
}
case lifecycle.Error:
apiErr = Error{
Code: "InvalidRequest",
Description: e.Error(),
HTTPStatusCode: http.StatusBadRequest,
}
// case lifecycle.Error:
// apiErr = Error{
// Code: "InvalidRequest",
// Description: e.Error(),
// HTTPStatusCode: http.StatusBadRequest,
// }
case tags.Error:
apiErr = Error{
Code: e.Code(),
Description: e.Error(),
HTTPStatusCode: http.StatusBadRequest,
}
case policy.Error:
apiErr = Error{
Code: "MalformedPolicy",
Description: e.Error(),
HTTPStatusCode: http.StatusBadRequest,
}
// case policy.Error:
// apiErr = Error{
// Code: "MalformedPolicy",
// Description: e.Error(),
// HTTPStatusCode: http.StatusBadRequest,
// }
case crypto.Error:
apiErr = Error{
Code: "XMinIOEncryptionError",
Description: e.Error(),
HTTPStatusCode: http.StatusBadRequest,
}
case minio.ErrorResponse:
case ErrorResponse:
apiErr = Error{
Code: e.Code,
Description: e.Message,
HTTPStatusCode: e.StatusCode,
}
case *googleapi.Error:
apiErr = Error{
Code: "XGCSInternalError",
Description: e.Message,
HTTPStatusCode: e.Code,
}
// GCS may send multiple errors, just pick the first one
// since S3 only sends one Error XML response.
if len(e.Errors) >= 1 {
apiErr.Code = e.Errors[0].Reason
}
case azblob.StorageError:
apiErr = Error{
Code: string(e.ServiceCode()),
Description: e.Error(),
HTTPStatusCode: e.Response().StatusCode,
}
// Add more Gateway SDKs here if any in future.
}
}

114
neofs/api/handler/api.go
View file

@ -1,131 +1,37 @@
package handler
import (
"context"
"crypto/ecdsa"
"math"
"errors"
"github.com/minio/minio/neofs/api"
"github.com/minio/minio/neofs/pool"
"github.com/nspcc-dev/neofs-api-go/refs"
"github.com/nspcc-dev/neofs-api-go/service"
"github.com/nspcc-dev/neofs-api-go/session"
crypto "github.com/nspcc-dev/neofs-crypto"
"github.com/pkg/errors"
"github.com/minio/minio/neofs/layer"
"go.uber.org/zap"
)
type (
handler struct {
log *zap.Logger
cli pool.Client
uid refs.OwnerID
tkn *service.Token
key *ecdsa.PrivateKey
obj layer.Client
}
Params struct {
Cli pool.Client
Log *zap.Logger
Key *ecdsa.PrivateKey
}
queryParams struct {
key *ecdsa.PrivateKey
addr refs.Address
verb service.Token_Info_Verb
Obj layer.Client
}
)
var _ api.Handler = (*handler)(nil)
func New(ctx context.Context, p Params) (api.Handler, error) {
var (
err error
uid refs.OwnerID
tkn *service.Token
)
func New(log *zap.Logger, obj layer.Client) (api.Handler, error) {
switch {
case p.Key == nil:
return nil, errors.New("empty private key")
case p.Cli == nil:
return nil, errors.New("empty gRPC client")
case p.Log == nil:
case obj == nil:
return nil, errors.New("empty NeoFS Object Layer")
case log == nil:
return nil, errors.New("empty logger")
}
if uid, err = refs.NewOwnerID(&p.Key.PublicKey); err != nil {
return nil, errors.Wrap(err, "could not fetch OwnerID")
} else if tkn, err = generateToken(ctx, p.Cli, p.Key); err != nil {
return nil, errors.Wrap(err, "could not prepare session token")
}
return &handler{
uid: uid,
tkn: tkn,
key: p.Key,
log: p.Log,
cli: p.Cli,
log: log,
obj: obj,
}, nil
}
func generateToken(ctx context.Context, cli pool.Client, key *ecdsa.PrivateKey) (*service.Token, error) {
owner, err := refs.NewOwnerID(&key.PublicKey)
if err != nil {
return nil, err
}
token := new(service.Token)
token.SetOwnerID(owner)
token.SetExpirationEpoch(math.MaxUint64)
token.SetOwnerKey(crypto.MarshalPublicKey(&key.PublicKey))
conn, err := cli.GetConnection(ctx)
if err != nil {
return nil, err
}
creator, err := session.NewGRPCCreator(conn, key)
if err != nil {
return nil, err
}
res, err := creator.Create(ctx, token)
if err != nil {
return nil, err
}
token.SetID(res.GetID())
token.SetSessionKey(res.GetSessionKey())
return token, nil
}
func prepareToken(t *service.Token, p queryParams) (*service.Token, error) {
sig := make([]byte, len(t.Signature))
copy(sig, t.Signature)
token := &service.Token{
Token_Info: service.Token_Info{
ID: t.ID,
OwnerID: t.OwnerID,
Verb: t.Verb,
Address: t.Address,
TokenLifetime: t.TokenLifetime,
SessionKey: t.SessionKey,
OwnerKey: t.OwnerKey,
},
Signature: sig,
}
token.SetAddress(p.addr)
token.SetVerb(p.verb)
err := service.AddSignatureWithKey(p.key, service.NewSignedSessionToken(token))
if err != nil {
return nil, err
}
return token, nil
}

171
neofs/api/handler/container.go
View file

@ -1,171 +0,0 @@
package handler
import (
"context"
"encoding/xml"
"net/http"
"time"
"github.com/minio/minio/auth"
"github.com/minio/minio/neofs/api"
"github.com/nspcc-dev/neofs-api-go/container"
"github.com/nspcc-dev/neofs-api-go/refs"
"github.com/nspcc-dev/neofs-api-go/service"
"github.com/pkg/errors"
"go.uber.org/zap"
"google.golang.org/grpc"
)
type (
// Owner - bucket owner/principal
Owner struct {
ID string
DisplayName string
}
// Bucket container for bucket metadata
Bucket struct {
CID refs.CID `xml:"-"` // ignored by response
Name string
CreationDate string // time string of format "2006-01-02T15:04:05.000Z"
}
// ListBucketsResponse - format for list buckets response
ListBucketsResponse struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListAllMyBucketsResult" json:"-"`
Owner Owner
// Container for one or more buckets.
Buckets struct {
Buckets []*Bucket `xml:"Bucket"`
} // Buckets are nested
}
cnrInfoParams struct {
cid refs.CID
tkn *service.BearerTokenMsg
}
)
func (h *handler) getContainerInfo(ctx context.Context, p cnrInfoParams) (*Bucket, error) {
var (
err error
con *grpc.ClientConn
res *container.GetResponse
)
req := new(container.GetRequest)
req.SetCID(p.cid)
req.SetTTL(service.SingleForwardingTTL)
req.SetBearer(p.tkn)
if con, err = h.cli.GetConnection(ctx); err != nil {
return nil, errors.Wrap(err, "could not fetch connection")
} else if err = service.SignRequestData(h.key, req); err != nil {
return nil, errors.Wrap(err, "could not sign container info request")
} else if res, err = container.NewServiceClient(con).Get(ctx, req); err != nil {
return nil, errors.Wrap(err, "could not fetch container info")
}
// TODO should extract nice name
// and datetime from container info:
_ = res
return &Bucket{
CID: p.cid,
Name: p.cid.String(),
CreationDate: new(time.Time).Format(time.RFC3339),
}, nil
}
func (h *handler) getContainerList(ctx context.Context, tkn *service.BearerTokenMsg) ([]*Bucket, error) {
var (
err error
inf *Bucket
con *grpc.ClientConn
res *container.ListResponse
)
req := new(container.ListRequest)
req.OwnerID = tkn.OwnerID
req.SetTTL(service.SingleForwardingTTL)
req.SetBearer(tkn)
if con, err = h.cli.GetConnection(ctx); err != nil {
return nil, errors.Wrap(err, "could not fetch connection")
} else if err = service.SignRequestData(h.key, req); err != nil {
return nil, errors.Wrap(err, "could not sign request")
} else if res, err = container.NewServiceClient(con).List(ctx, req); err != nil {
return nil, errors.Wrap(err, "could not fetch list containers")
}
params := cnrInfoParams{tkn: tkn}
result := make([]*Bucket, 0, len(res.CID))
for _, cid := range res.CID {
params.cid = cid
if inf, err = h.getContainerInfo(ctx, params); err != nil {
return nil, errors.Wrap(err, "could not fetch container info")
}
result = append(result, inf)
}
return result, nil
}
func (h *handler) ListBucketsHandler(w http.ResponseWriter, r *http.Request) {
var (
err error
lst []*Bucket
tkn *service.BearerTokenMsg
)
// TODO think about deadlines
ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second)
defer cancel()
if tkn, err = auth.GetBearerToken(ctx); err != nil {
h.log.Error("could not fetch bearer token",
zap.Error(err))
e := api.GetAPIError(api.ErrInternalError)
api.WriteErrorResponse(ctx, w, api.Error{
Code: e.Code,
Description: err.Error(),
HTTPStatusCode: e.HTTPStatusCode,
}, r.URL)
return
} else if lst, err = h.getContainerList(ctx, tkn); err != nil {
h.log.Error("could not fetch bearer token",
zap.Error(err))
// TODO check that error isn't gRPC error
e := api.GetAPIError(api.ErrInternalError)
api.WriteErrorResponse(ctx, w, api.Error{
Code: e.Code,
Description: err.Error(),
HTTPStatusCode: e.HTTPStatusCode,
}, r.URL)
return
}
result := &ListBucketsResponse{Owner: Owner{
ID: tkn.OwnerID.String(),
DisplayName: tkn.OwnerID.String(),
}}
result.Buckets.Buckets = lst
// Generate response.
encodedSuccessResponse := api.EncodeResponse(result)
// Write response.
api.WriteSuccessResponseXML(w, encodedSuccessResponse)
}

8
neofs/api/handler/unimplemented.go
View file

@ -6,6 +6,14 @@ import (
"github.com/minio/minio/neofs/api"
)
func (h *handler) ListBucketsHandler(w http.ResponseWriter, r *http.Request) {
api.WriteErrorResponse(r.Context(), w, api.Error{
Code: "XNeoFSUnimplemented",
Description: "implement me ListBucketsHandler",
HTTPStatusCode: http.StatusNotImplemented,
}, r.URL)
}
func (h *handler) HeadObjectHandler(w http.ResponseWriter, r *http.Request) {
api.WriteErrorResponse(r.Context(), w, api.Error{
Code: "XNeoFSUnimplemented",

68
neofs/api/response.go
View file

@ -14,7 +14,7 @@ import (
)
type (
// APIErrorResponse - error response format
// ErrorResponse - error response format
ErrorResponse struct {
XMLName xml.Name `xml:"Error" json:"-"`
Code string
@ -22,9 +22,18 @@ type (
Key string `xml:"Key,omitempty" json:"Key,omitempty"`
BucketName string `xml:"BucketName,omitempty" json:"BucketName,omitempty"`
Resource string
Region string `xml:"Region,omitempty" json:"Region,omitempty"`
RequestID string `xml:"RequestId" json:"RequestId"`
HostID string `xml:"HostId" json:"HostId"`
// Region where the bucket is located. This header is returned
// only in HEAD bucket and ListObjects response.
Region string `xml:"Region,omitempty" json:"Region,omitempty"`
// Captures the server string returned in response header.
Server string `xml:"-" json:"-"`
// Underlying HTTP status code for the returned error
StatusCode int `xml:"-" json:"-"`
}
// APIError structure
@ -61,6 +70,49 @@ const (
var deploymentID, _ = uuid.NewRandom()
// Non exhaustive list of AWS S3 standard error responses -
// http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
var s3ErrorResponseMap = map[string]string{
"AccessDenied": "Access Denied.",
"BadDigest": "The Content-Md5 you specified did not match what we received.",
"EntityTooSmall": "Your proposed upload is smaller than the minimum allowed object size.",
"EntityTooLarge": "Your proposed upload exceeds the maximum allowed object size.",
"IncompleteBody": "You did not provide the number of bytes specified by the Content-Length HTTP header.",
"InternalError": "We encountered an internal error, please try again.",
"InvalidAccessKeyId": "The access key ID you provided does not exist in our records.",
"InvalidBucketName": "The specified bucket is not valid.",
"InvalidDigest": "The Content-Md5 you specified is not valid.",
"InvalidRange": "The requested range is not satisfiable",
"MalformedXML": "The XML you provided was not well-formed or did not validate against our published schema.",
"MissingContentLength": "You must provide the Content-Length HTTP header.",
"MissingContentMD5": "Missing required header for this request: Content-Md5.",
"MissingRequestBodyError": "Request body is empty.",
"NoSuchBucket": "The specified bucket does not exist.",
"NoSuchBucketPolicy": "The bucket policy does not exist",
"NoSuchKey": "The specified key does not exist.",
"NoSuchUpload": "The specified multipart upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed.",
"NotImplemented": "A header you provided implies functionality that is not implemented",
"PreconditionFailed": "At least one of the pre-conditions you specified did not hold",
"RequestTimeTooSkewed": "The difference between the request time and the server's time is too large.",
"SignatureDoesNotMatch": "The request signature we calculated does not match the signature you provided. Check your key and signing method.",
"MethodNotAllowed": "The specified method is not allowed against this resource.",
"InvalidPart": "One or more of the specified parts could not be found.",
"InvalidPartOrder": "The list of parts was not in ascending order. The parts list must be specified in order by part number.",
"InvalidObjectState": "The operation is not valid for the current state of the object.",
"AuthorizationHeaderMalformed": "The authorization header is malformed; the region is wrong.",
"MalformedPOSTRequest": "The body of your POST request is not well-formed multipart/form-data.",
"BucketNotEmpty": "The bucket you tried to delete is not empty",
"AllAccessDisabled": "All access to this bucket has been disabled.",
"MalformedPolicy": "Policy has invalid resource.",
"MissingFields": "Missing fields in request.",
"AuthorizationQueryParametersError": "Error parsing the X-Amz-Credential parameter; the Credential is mal-formed; expecting \"<YOUR-AKID>/YYYYMMDD/REGION/SERVICE/aws4_request\".",
"MalformedDate": "Invalid date format header, expected to be in ISO8601, RFC1123 or RFC1123Z time format.",
"BucketAlreadyOwnedByYou": "Your previous request to create the named bucket succeeded and you already own it.",
"InvalidDuration": "Duration provided in the request is invalid.",
"XAmzContentSHA256Mismatch": "The provided 'x-amz-content-sha256' header does not match what was computed.",
// Add new API errors here.
}
// WriteErrorResponse writes error headers
func WriteErrorResponse(ctx context.Context, w http.ResponseWriter, err Error, reqURL *url.URL) {
switch err.Code {
@ -134,3 +186,15 @@ func EncodeResponse(response interface{}) []byte {
func WriteSuccessResponseXML(w http.ResponseWriter, response []byte) {
writeResponse(w, http.StatusOK, response, mimeXML)
}
// Error - Returns S3 error string.
func (e ErrorResponse) Error() string {
if e.Message == "" {
msg, ok := s3ErrorResponseMap[e.Code]
if !ok {
msg = fmt.Sprintf("Error response code %s.", e.Code)
}
return msg
}
return e.Message
}