[#219] Return ETag in quotes #255
17 changed files with 97 additions and 57 deletions
|
@ -54,6 +54,7 @@ This document outlines major changes between releases.
|
||||||
- Apply placement policies and copies if there is at least one valid value (#168)
|
- Apply placement policies and copies if there is at least one valid value (#168)
|
||||||
- Generalise config param `use_default_xmlns_for_complete_multipart` to `use_default_xmlns` so that use default xmlns for all requests (#221)
|
- Generalise config param `use_default_xmlns_for_complete_multipart` to `use_default_xmlns` so that use default xmlns for all requests (#221)
|
||||||
- Set server IdleTimeout and ReadHeaderTimeout to `30s` and allow to configure them (#220)
|
- Set server IdleTimeout and ReadHeaderTimeout to `30s` and allow to configure them (#220)
|
||||||
|
- Return `ETag` value in quotes (#219)
|
||||||
|
|
||||||
### Removed
|
### Removed
|
||||||
- Drop `tree.service` param (now endpoints from `peers` section are used) (#133)
|
- Drop `tree.service` param (now endpoints from `peers` section are used) (#133)
|
||||||
|
|
|
@ -2,6 +2,7 @@ package data
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/xml"
|
"encoding/xml"
|
||||||
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
|
@ -83,12 +84,12 @@ type (
|
||||||
)
|
)
|
||||||
|
|
||||||
// NotificationInfoFromObject creates new NotificationInfo from ObjectInfo.
|
// NotificationInfoFromObject creates new NotificationInfo from ObjectInfo.
|
||||||
func NotificationInfoFromObject(objInfo *ObjectInfo) *NotificationInfo {
|
func NotificationInfoFromObject(objInfo *ObjectInfo, md5Enabled bool) *NotificationInfo {
|
||||||
return &NotificationInfo{
|
return &NotificationInfo{
|
||||||
Name: objInfo.Name,
|
Name: objInfo.Name,
|
||||||
Version: objInfo.VersionID(),
|
Version: objInfo.VersionID(),
|
||||||
Size: objInfo.Size,
|
Size: objInfo.Size,
|
||||||
HashSum: objInfo.HashSum,
|
HashSum: Quote(objInfo.ETag(md5Enabled)),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -135,3 +136,11 @@ func (b BucketSettings) VersioningEnabled() bool {
|
||||||
func (b BucketSettings) VersioningSuspended() bool {
|
func (b BucketSettings) VersioningSuspended() bool {
|
||||||
return b.Versioning == VersioningSuspended
|
return b.Versioning == VersioningSuspended
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func Quote(val string) string {
|
||||||
|
return "\"" + val + "\""
|
||||||
|
}
|
||||||
|
|
||||||
|
func UnQuote(val string) string {
|
||||||
|
return strings.Trim(val, "\"")
|
||||||
|
}
|
||||||
|
|
|
@ -93,9 +93,17 @@ type PartInfo struct {
|
||||||
|
|
||||||
// ToHeaderString form short part representation to use in S3-Completed-Parts header.
|
// ToHeaderString form short part representation to use in S3-Completed-Parts header.
|
||||||
func (p *PartInfo) ToHeaderString() string {
|
func (p *PartInfo) ToHeaderString() string {
|
||||||
|
// ETag value contains SHA256 checksum which is used while getting object parts attributes.
|
||||||
return strconv.Itoa(p.Number) + "-" + strconv.FormatUint(p.Size, 10) + "-" + p.ETag
|
return strconv.Itoa(p.Number) + "-" + strconv.FormatUint(p.Size, 10) + "-" + p.ETag
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (p *PartInfo) GetETag(md5Enabled bool) string {
|
||||||
|
if md5Enabled && len(p.MD5) > 0 {
|
||||||
|
return p.MD5
|
||||||
|
}
|
||||||
|
return p.ETag
|
||||||
|
}
|
||||||
|
|
||||||
// LockInfo is lock information to create appropriate tree node.
|
// LockInfo is lock information to create appropriate tree node.
|
||||||
type LockInfo struct {
|
type LockInfo struct {
|
||||||
id uint64
|
id uint64
|
||||||
|
|
|
@ -465,7 +465,7 @@ func (h *handler) PutObjectACLHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
if updated {
|
if updated {
|
||||||
s := &SendNotificationParams{
|
s := &SendNotificationParams{
|
||||||
Event: EventObjectACLPut,
|
Event: EventObjectACLPut,
|
||||||
NotificationInfo: data.NotificationInfoFromObject(objInfo),
|
NotificationInfo: data.NotificationInfoFromObject(objInfo, h.cfg.MD5Enabled()),
|
||||||
BktInfo: bktInfo,
|
BktInfo: bktInfo,
|
||||||
ReqInfo: reqInfo,
|
ReqInfo: reqInfo,
|
||||||
}
|
}
|
||||||
|
@ -642,7 +642,7 @@ func parseGrantee(grantees string) ([]*Grantee, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func formGrantee(granteeType, value string) (*Grantee, error) {
|
func formGrantee(granteeType, value string) (*Grantee, error) {
|
||||||
value = strings.Trim(value, "\"")
|
value = data.UnQuote(value)
|
||||||
switch granteeType {
|
switch granteeType {
|
||||||
case "id":
|
case "id":
|
||||||
return &Grantee{
|
return &Grantee{
|
||||||
|
|
|
@ -1,6 +1,8 @@
|
||||||
package handler
|
package handler
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/hex"
|
||||||
"fmt"
|
"fmt"
|
||||||
"net/http"
|
"net/http"
|
||||||
"strconv"
|
"strconv"
|
||||||
|
@ -106,7 +108,7 @@ func (h *handler) GetObjectAttributesHandler(w http.ResponseWriter, r *http.Requ
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if err = checkPreconditions(info, params.Conditional); err != nil {
|
if err = checkPreconditions(info, params.Conditional, h.cfg.MD5Enabled()); err != nil {
|
||||||
h.logAndSendError(w, "precondition failed", reqInfo, err)
|
h.logAndSendError(w, "precondition failed", reqInfo, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
@ -117,7 +119,7 @@ func (h *handler) GetObjectAttributesHandler(w http.ResponseWriter, r *http.Requ
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
response, err := encodeToObjectAttributesResponse(info, params)
|
response, err := encodeToObjectAttributesResponse(info, params, h.cfg.MD5Enabled())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.logAndSendError(w, "couldn't encode object info to response", reqInfo, err)
|
h.logAndSendError(w, "couldn't encode object info to response", reqInfo, err)
|
||||||
return
|
return
|
||||||
|
@ -179,19 +181,23 @@ func parseGetObjectAttributeArgs(r *http.Request) (*GetObjectAttributesArgs, err
|
||||||
return res, err
|
return res, err
|
||||||
}
|
}
|
||||||
|
|
||||||
func encodeToObjectAttributesResponse(info *data.ObjectInfo, p *GetObjectAttributesArgs) (*GetObjectAttributesResponse, error) {
|
func encodeToObjectAttributesResponse(info *data.ObjectInfo, p *GetObjectAttributesArgs, md5Enabled bool) (*GetObjectAttributesResponse, error) {
|
||||||
resp := &GetObjectAttributesResponse{}
|
resp := &GetObjectAttributesResponse{}
|
||||||
|
|
||||||
for _, attr := range p.Attributes {
|
for _, attr := range p.Attributes {
|
||||||
switch attr {
|
switch attr {
|
||||||
case eTag:
|
case eTag:
|
||||||
resp.ETag = info.HashSum
|
resp.ETag = data.Quote(info.ETag(md5Enabled))
|
||||||
case storageClass:
|
case storageClass:
|
||||||
resp.StorageClass = api.DefaultStorageClass
|
resp.StorageClass = api.DefaultStorageClass
|
||||||
case objectSize:
|
case objectSize:
|
||||||
resp.ObjectSize = info.Size
|
resp.ObjectSize = info.Size
|
||||||
case checksum:
|
case checksum:
|
||||||
resp.Checksum = &Checksum{ChecksumSHA256: info.HashSum}
|
checksumBytes, err := hex.DecodeString(info.HashSum)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("form upload attributes: %w", err)
|
||||||
|
}
|
||||||
|
resp.Checksum = &Checksum{ChecksumSHA256: base64.StdEncoding.EncodeToString(checksumBytes)}
|
||||||
case objectParts:
|
case objectParts:
|
||||||
parts, err := formUploadAttributes(info, p.MaxParts, p.PartNumberMarker)
|
parts, err := formUploadAttributes(info, p.MaxParts, p.PartNumberMarker)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -219,10 +225,15 @@ func formUploadAttributes(info *data.ObjectInfo, maxParts, marker int) (*ObjectP
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("invalid completed part: %w", err)
|
return nil, fmt.Errorf("invalid completed part: %w", err)
|
||||||
}
|
}
|
||||||
|
// ETag value contains SHA256 checksum.
|
||||||
|
checksumBytes, err := hex.DecodeString(part.ETag)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("invalid sha256 checksum in completed part: %w", err)
|
||||||
|
}
|
||||||
parts[i] = Part{
|
parts[i] = Part{
|
||||||
PartNumber: part.PartNumber,
|
PartNumber: part.PartNumber,
|
||||||
Size: int(part.Size),
|
Size: int(part.Size),
|
||||||
ChecksumSHA256: part.ETag,
|
ChecksumSHA256: base64.StdEncoding.EncodeToString(checksumBytes),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -1,6 +1,8 @@
|
||||||
package handler
|
package handler
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/hex"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
@ -24,11 +26,13 @@ func TestGetObjectPartsAttributes(t *testing.T) {
|
||||||
multipartUpload := createMultipartUpload(hc, bktName, objMultipartName, map[string]string{})
|
multipartUpload := createMultipartUpload(hc, bktName, objMultipartName, map[string]string{})
|
||||||
etag, _ := uploadPart(hc, bktName, objMultipartName, multipartUpload.UploadID, 1, partSize)
|
etag, _ := uploadPart(hc, bktName, objMultipartName, multipartUpload.UploadID, 1, partSize)
|
||||||
completeMultipartUpload(hc, bktName, objMultipartName, multipartUpload.UploadID, []string{etag})
|
completeMultipartUpload(hc, bktName, objMultipartName, multipartUpload.UploadID, []string{etag})
|
||||||
|
etagBytes, err := hex.DecodeString(etag[1 : len(etag)-1])
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
result = getObjectAttributes(hc, bktName, objMultipartName, objectParts)
|
result = getObjectAttributes(hc, bktName, objMultipartName, objectParts)
|
||||||
require.NotNil(t, result.ObjectParts)
|
require.NotNil(t, result.ObjectParts)
|
||||||
require.Len(t, result.ObjectParts.Parts, 1)
|
require.Len(t, result.ObjectParts.Parts, 1)
|
||||||
require.Equal(t, etag, result.ObjectParts.Parts[0].ChecksumSHA256)
|
require.Equal(t, base64.StdEncoding.EncodeToString(etagBytes), result.ObjectParts.Parts[0].ChecksumSHA256)
|
||||||
require.Equal(t, partSize, result.ObjectParts.Parts[0].Size)
|
require.Equal(t, partSize, result.ObjectParts.Parts[0].Size)
|
||||||
require.Equal(t, 1, result.ObjectParts.PartsCount)
|
require.Equal(t, 1, result.ObjectParts.PartsCount)
|
||||||
}
|
}
|
||||||
|
|
|
@ -177,7 +177,7 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if err = checkPreconditions(srcObjInfo, args.Conditional); err != nil {
|
if err = checkPreconditions(srcObjInfo, args.Conditional, h.cfg.MD5Enabled()); err != nil {
|
||||||
h.logAndSendError(w, "precondition failed", reqInfo, errors.GetAPIError(errors.ErrPreconditionFailed))
|
h.logAndSendError(w, "precondition failed", reqInfo, errors.GetAPIError(errors.ErrPreconditionFailed))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
@ -224,7 +224,10 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
}
|
}
|
||||||
dstObjInfo := extendedDstObjInfo.ObjectInfo
|
dstObjInfo := extendedDstObjInfo.ObjectInfo
|
||||||
|
|
||||||
if err = middleware.EncodeToResponse(w, &CopyObjectResponse{LastModified: dstObjInfo.Created.UTC().Format(time.RFC3339), ETag: dstObjInfo.HashSum}); err != nil {
|
if err = middleware.EncodeToResponse(w, &CopyObjectResponse{
|
||||||
|
LastModified: dstObjInfo.Created.UTC().Format(time.RFC3339),
|
||||||
|
ETag: data.Quote(dstObjInfo.ETag(h.cfg.MD5Enabled())),
|
||||||
|
}); err != nil {
|
||||||
h.logAndSendError(w, "something went wrong", reqInfo, err, additional...)
|
h.logAndSendError(w, "something went wrong", reqInfo, err, additional...)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
@ -268,7 +271,7 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
|
|
||||||
s := &SendNotificationParams{
|
s := &SendNotificationParams{
|
||||||
Event: EventObjectCreatedCopy,
|
Event: EventObjectCreatedCopy,
|
||||||
NotificationInfo: data.NotificationInfoFromObject(dstObjInfo),
|
NotificationInfo: data.NotificationInfoFromObject(dstObjInfo, h.cfg.MD5Enabled()),
|
||||||
BktInfo: dstBktInfo,
|
BktInfo: dstBktInfo,
|
||||||
ReqInfo: reqInfo,
|
ReqInfo: reqInfo,
|
||||||
}
|
}
|
||||||
|
@ -311,8 +314,8 @@ func isCopyingToItselfForbidden(reqInfo *middleware.ReqInfo, srcBucket string, s
|
||||||
func parseCopyObjectArgs(headers http.Header) (*copyObjectArgs, error) {
|
func parseCopyObjectArgs(headers http.Header) (*copyObjectArgs, error) {
|
||||||
var err error
|
var err error
|
||||||
args := &conditionalArgs{
|
args := &conditionalArgs{
|
||||||
IfMatch: headers.Get(api.AmzCopyIfMatch),
|
IfMatch: data.UnQuote(headers.Get(api.AmzCopyIfMatch)),
|
||||||
IfNoneMatch: headers.Get(api.AmzCopyIfNoneMatch),
|
IfNoneMatch: data.UnQuote(headers.Get(api.AmzCopyIfNoneMatch)),
|
||||||
}
|
}
|
||||||
|
|
||||||
if args.IfModifiedSince, err = parseHTTPTime(headers.Get(api.AmzCopyIfModifiedSince)); err != nil {
|
if args.IfModifiedSince, err = parseHTTPTime(headers.Get(api.AmzCopyIfModifiedSince)); err != nil {
|
||||||
|
|
|
@ -95,7 +95,7 @@ func writeHeaders(h http.Header, requestHeader http.Header, extendedInfo *data.E
|
||||||
h.Set(api.ContentLength, strconv.FormatUint(info.Size, 10))
|
h.Set(api.ContentLength, strconv.FormatUint(info.Size, 10))
|
||||||
}
|
}
|
||||||
|
|
||||||
h.Set(api.ETag, info.ETag(md5Enabled))
|
h.Set(api.ETag, data.Quote(info.ETag(md5Enabled)))
|
||||||
|
|
||||||
h.Set(api.AmzTaggingCount, strconv.Itoa(tagSetLength))
|
h.Set(api.AmzTaggingCount, strconv.Itoa(tagSetLength))
|
||||||
h.Set(api.AmzStorageClass, api.DefaultStorageClass)
|
h.Set(api.AmzStorageClass, api.DefaultStorageClass)
|
||||||
|
@ -157,7 +157,7 @@ func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
}
|
}
|
||||||
info := extendedInfo.ObjectInfo
|
info := extendedInfo.ObjectInfo
|
||||||
|
|
||||||
if err = checkPreconditions(info, conditional); err != nil {
|
if err = checkPreconditions(info, conditional, h.cfg.MD5Enabled()); err != nil {
|
||||||
h.logAndSendError(w, "precondition failed", reqInfo, err)
|
h.logAndSendError(w, "precondition failed", reqInfo, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
@ -238,12 +238,13 @@ func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func checkPreconditions(info *data.ObjectInfo, args *conditionalArgs) error {
|
func checkPreconditions(info *data.ObjectInfo, args *conditionalArgs, md5Enabled bool) error {
|
||||||
if len(args.IfMatch) > 0 && args.IfMatch != info.HashSum {
|
etag := info.ETag(md5Enabled)
|
||||||
return fmt.Errorf("%w: etag mismatched: '%s', '%s'", errors.GetAPIError(errors.ErrPreconditionFailed), args.IfMatch, info.HashSum)
|
if len(args.IfMatch) > 0 && args.IfMatch != etag {
|
||||||
|
return fmt.Errorf("%w: etag mismatched: '%s', '%s'", errors.GetAPIError(errors.ErrPreconditionFailed), args.IfMatch, etag)
|
||||||
}
|
}
|
||||||
if len(args.IfNoneMatch) > 0 && args.IfNoneMatch == info.HashSum {
|
if len(args.IfNoneMatch) > 0 && args.IfNoneMatch == etag {
|
||||||
return fmt.Errorf("%w: etag matched: '%s', '%s'", errors.GetAPIError(errors.ErrNotModified), args.IfNoneMatch, info.HashSum)
|
return fmt.Errorf("%w: etag matched: '%s', '%s'", errors.GetAPIError(errors.ErrNotModified), args.IfNoneMatch, etag)
|
||||||
}
|
}
|
||||||
if args.IfModifiedSince != nil && info.Created.Before(*args.IfModifiedSince) {
|
if args.IfModifiedSince != nil && info.Created.Before(*args.IfModifiedSince) {
|
||||||
return fmt.Errorf("%w: not modified since '%s', last modified '%s'", errors.GetAPIError(errors.ErrNotModified),
|
return fmt.Errorf("%w: not modified since '%s', last modified '%s'", errors.GetAPIError(errors.ErrNotModified),
|
||||||
|
@ -267,8 +268,8 @@ func checkPreconditions(info *data.ObjectInfo, args *conditionalArgs) error {
|
||||||
func parseConditionalHeaders(headers http.Header) (*conditionalArgs, error) {
|
func parseConditionalHeaders(headers http.Header) (*conditionalArgs, error) {
|
||||||
var err error
|
var err error
|
||||||
args := &conditionalArgs{
|
args := &conditionalArgs{
|
||||||
IfMatch: strings.Trim(headers.Get(api.IfMatch), "\""),
|
IfMatch: data.UnQuote(headers.Get(api.IfMatch)),
|
||||||
IfNoneMatch: strings.Trim(headers.Get(api.IfNoneMatch), "\""),
|
IfNoneMatch: data.UnQuote(headers.Get(api.IfNoneMatch)),
|
||||||
}
|
}
|
||||||
|
|
||||||
if args.IfModifiedSince, err = parseHTTPTime(headers.Get(api.IfModifiedSince)); err != nil {
|
if args.IfModifiedSince, err = parseHTTPTime(headers.Get(api.IfModifiedSince)); err != nil {
|
||||||
|
|
|
@ -147,7 +147,7 @@ func TestPreconditions(t *testing.T) {
|
||||||
},
|
},
|
||||||
} {
|
} {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
actual := checkPreconditions(tc.info, tc.args)
|
actual := checkPreconditions(tc.info, tc.args, false)
|
||||||
if tc.expected == nil {
|
if tc.expected == nil {
|
||||||
require.NoError(t, actual)
|
require.NoError(t, actual)
|
||||||
} else {
|
} else {
|
||||||
|
@ -203,11 +203,11 @@ func TestGetObjectEnabledMD5(t *testing.T) {
|
||||||
_, objInfo := createBucketAndObject(hc, bktName, objName)
|
_, objInfo := createBucketAndObject(hc, bktName, objName)
|
||||||
|
|
||||||
_, headers := getObject(hc, bktName, objName)
|
_, headers := getObject(hc, bktName, objName)
|
||||||
require.Equal(t, objInfo.HashSum, headers.Get(api.ETag))
|
require.Equal(t, data.Quote(objInfo.HashSum), headers.Get(api.ETag))
|
||||||
|
|
||||||
hc.config.md5Enabled = true
|
hc.config.md5Enabled = true
|
||||||
_, headers = getObject(hc, bktName, objName)
|
_, headers = getObject(hc, bktName, objName)
|
||||||
require.Equal(t, objInfo.MD5Sum, headers.Get(api.ETag))
|
require.Equal(t, data.Quote(objInfo.MD5Sum), headers.Get(api.ETag))
|
||||||
}
|
}
|
||||||
|
|
||||||
func putObjectContent(hc *handlerContext, bktName, objName, content string) {
|
func putObjectContent(hc *handlerContext, bktName, objName, content string) {
|
||||||
|
|
|
@ -65,7 +65,7 @@ func (h *handler) HeadObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if err = checkPreconditions(info, conditional); err != nil {
|
if err = checkPreconditions(info, conditional, h.cfg.MD5Enabled()); err != nil {
|
||||||
h.logAndSendError(w, "precondition failed", reqInfo, err)
|
h.logAndSendError(w, "precondition failed", reqInfo, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
|
@ -264,7 +264,7 @@ func (h *handler) UploadPartHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
addSSECHeaders(w.Header(), r.Header)
|
addSSECHeaders(w.Header(), r.Header)
|
||||||
}
|
}
|
||||||
|
|
||||||
w.Header().Set(api.ETag, hash)
|
w.Header().Set(api.ETag, data.Quote(hash))
|
||||||
middleware.WriteSuccessResponseHeadersOnly(w)
|
middleware.WriteSuccessResponseHeadersOnly(w)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -339,7 +339,7 @@ func (h *handler) UploadPartCopy(w http.ResponseWriter, r *http.Request) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if err = checkPreconditions(srcInfo, args.Conditional); err != nil {
|
if err = checkPreconditions(srcInfo, args.Conditional, h.cfg.MD5Enabled()); err != nil {
|
||||||
h.logAndSendError(w, "precondition failed", reqInfo, errors.GetAPIError(errors.ErrPreconditionFailed),
|
h.logAndSendError(w, "precondition failed", reqInfo, errors.GetAPIError(errors.ErrPreconditionFailed),
|
||||||
additional...)
|
additional...)
|
||||||
return
|
return
|
||||||
|
@ -384,7 +384,7 @@ func (h *handler) UploadPartCopy(w http.ResponseWriter, r *http.Request) {
|
||||||
|
|
||||||
response := UploadPartCopyResponse{
|
response := UploadPartCopyResponse{
|
||||||
LastModified: info.Created.UTC().Format(time.RFC3339),
|
LastModified: info.Created.UTC().Format(time.RFC3339),
|
||||||
ETag: info.ETag(h.cfg.MD5Enabled()),
|
ETag: data.Quote(info.ETag(h.cfg.MD5Enabled())),
|
||||||
}
|
}
|
||||||
|
|
||||||
if p.Info.Encryption.Enabled() {
|
if p.Info.Encryption.Enabled() {
|
||||||
|
@ -449,7 +449,7 @@ func (h *handler) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.
|
||||||
response := CompleteMultipartUploadResponse{
|
response := CompleteMultipartUploadResponse{
|
||||||
Bucket: objInfo.Bucket,
|
Bucket: objInfo.Bucket,
|
||||||
Key: objInfo.Name,
|
Key: objInfo.Name,
|
||||||
ETag: objInfo.ETag(h.cfg.MD5Enabled()),
|
ETag: data.Quote(objInfo.ETag(h.cfg.MD5Enabled())),
|
||||||
}
|
}
|
||||||
|
|
||||||
if settings.VersioningEnabled() {
|
if settings.VersioningEnabled() {
|
||||||
|
@ -513,7 +513,7 @@ func (h *handler) completeMultipartUpload(r *http.Request, c *layer.CompleteMult
|
||||||
|
|
||||||
s := &SendNotificationParams{
|
s := &SendNotificationParams{
|
||||||
Event: EventObjectCreatedCompleteMultipartUpload,
|
Event: EventObjectCreatedCompleteMultipartUpload,
|
||||||
NotificationInfo: data.NotificationInfoFromObject(objInfo),
|
NotificationInfo: data.NotificationInfoFromObject(objInfo, h.cfg.MD5Enabled()),
|
||||||
BktInfo: bktInfo,
|
BktInfo: bktInfo,
|
||||||
ReqInfo: reqInfo,
|
ReqInfo: reqInfo,
|
||||||
}
|
}
|
||||||
|
|
|
@ -13,6 +13,7 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
s3Errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
s3Errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
|
||||||
|
@ -298,11 +299,11 @@ func TestMultipartUploadEnabledMD5(t *testing.T) {
|
||||||
multipartUpload := createMultipartUpload(hc, bktName, objName, map[string]string{})
|
multipartUpload := createMultipartUpload(hc, bktName, objName, map[string]string{})
|
||||||
etag1, partBody1 := uploadPart(hc, bktName, objName, multipartUpload.UploadID, 1, partSize)
|
etag1, partBody1 := uploadPart(hc, bktName, objName, multipartUpload.UploadID, 1, partSize)
|
||||||
md5Sum1 := md5.Sum(partBody1)
|
md5Sum1 := md5.Sum(partBody1)
|
||||||
require.Equal(t, hex.EncodeToString(md5Sum1[:]), etag1)
|
require.Equal(t, data.Quote(hex.EncodeToString(md5Sum1[:])), etag1)
|
||||||
|
|
||||||
etag2, partBody2 := uploadPart(hc, bktName, objName, multipartUpload.UploadID, 2, partSize)
|
etag2, partBody2 := uploadPart(hc, bktName, objName, multipartUpload.UploadID, 2, partSize)
|
||||||
md5Sum2 := md5.Sum(partBody2)
|
md5Sum2 := md5.Sum(partBody2)
|
||||||
require.Equal(t, hex.EncodeToString(md5Sum2[:]), etag2)
|
require.Equal(t, data.Quote(hex.EncodeToString(md5Sum2[:])), etag2)
|
||||||
|
|
||||||
w := completeMultipartUploadBase(hc, bktName, objName, multipartUpload.UploadID, []string{etag1, etag2})
|
w := completeMultipartUploadBase(hc, bktName, objName, multipartUpload.UploadID, []string{etag1, etag2})
|
||||||
assertStatus(t, w, http.StatusOK)
|
assertStatus(t, w, http.StatusOK)
|
||||||
|
@ -310,7 +311,7 @@ func TestMultipartUploadEnabledMD5(t *testing.T) {
|
||||||
err := xml.NewDecoder(w.Result().Body).Decode(resp)
|
err := xml.NewDecoder(w.Result().Body).Decode(resp)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
completeMD5Sum := md5.Sum(append(md5Sum1[:], md5Sum2[:]...))
|
completeMD5Sum := md5.Sum(append(md5Sum1[:], md5Sum2[:]...))
|
||||||
require.Equal(t, hex.EncodeToString(completeMD5Sum[:])+"-2", resp.ETag)
|
require.Equal(t, data.Quote(hex.EncodeToString(completeMD5Sum[:])+"-2"), resp.ETag)
|
||||||
}
|
}
|
||||||
|
|
||||||
func uploadPartCopy(hc *handlerContext, bktName, objName, uploadID string, num int, srcObj string, start, end int) *UploadPartCopyResponse {
|
func uploadPartCopy(hc *handlerContext, bktName, objName, uploadID string, num int, srcObj string, start, end int) *UploadPartCopyResponse {
|
||||||
|
|
|
@ -34,12 +34,12 @@ func (h *handler) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if err = middleware.EncodeToResponse(w, encodeV1(params, list)); err != nil {
|
if err = middleware.EncodeToResponse(w, h.encodeV1(params, list)); err != nil {
|
||||||
h.logAndSendError(w, "something went wrong", reqInfo, err)
|
h.logAndSendError(w, "something went wrong", reqInfo, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func encodeV1(p *layer.ListObjectsParamsV1, list *layer.ListObjectsInfoV1) *ListObjectsV1Response {
|
func (h *handler) encodeV1(p *layer.ListObjectsParamsV1, list *layer.ListObjectsInfoV1) *ListObjectsV1Response {
|
||||||
res := &ListObjectsV1Response{
|
res := &ListObjectsV1Response{
|
||||||
Name: p.BktInfo.Name,
|
Name: p.BktInfo.Name,
|
||||||
EncodingType: p.Encode,
|
EncodingType: p.Encode,
|
||||||
|
@ -53,7 +53,7 @@ func encodeV1(p *layer.ListObjectsParamsV1, list *layer.ListObjectsInfoV1) *List
|
||||||
|
|
||||||
res.CommonPrefixes = fillPrefixes(list.Prefixes, p.Encode)
|
res.CommonPrefixes = fillPrefixes(list.Prefixes, p.Encode)
|
||||||
|
|
||||||
res.Contents = fillContentsWithOwner(list.Objects, p.Encode)
|
res.Contents = fillContentsWithOwner(list.Objects, p.Encode, h.cfg.MD5Enabled())
|
||||||
|
|
||||||
return res
|
return res
|
||||||
}
|
}
|
||||||
|
@ -78,12 +78,12 @@ func (h *handler) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if err = middleware.EncodeToResponse(w, encodeV2(params, list)); err != nil {
|
if err = middleware.EncodeToResponse(w, h.encodeV2(params, list)); err != nil {
|
||||||
h.logAndSendError(w, "something went wrong", reqInfo, err)
|
h.logAndSendError(w, "something went wrong", reqInfo, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func encodeV2(p *layer.ListObjectsParamsV2, list *layer.ListObjectsInfoV2) *ListObjectsV2Response {
|
func (h *handler) encodeV2(p *layer.ListObjectsParamsV2, list *layer.ListObjectsInfoV2) *ListObjectsV2Response {
|
||||||
res := &ListObjectsV2Response{
|
res := &ListObjectsV2Response{
|
||||||
Name: p.BktInfo.Name,
|
Name: p.BktInfo.Name,
|
||||||
EncodingType: p.Encode,
|
EncodingType: p.Encode,
|
||||||
|
@ -99,7 +99,7 @@ func encodeV2(p *layer.ListObjectsParamsV2, list *layer.ListObjectsInfoV2) *List
|
||||||
|
|
||||||
res.CommonPrefixes = fillPrefixes(list.Prefixes, p.Encode)
|
res.CommonPrefixes = fillPrefixes(list.Prefixes, p.Encode)
|
||||||
|
|
||||||
res.Contents = fillContents(list.Objects, p.Encode, p.FetchOwner)
|
res.Contents = fillContents(list.Objects, p.Encode, p.FetchOwner, h.cfg.MD5Enabled())
|
||||||
|
|
||||||
return res
|
return res
|
||||||
}
|
}
|
||||||
|
@ -185,18 +185,18 @@ func fillPrefixes(src []string, encode string) []CommonPrefix {
|
||||||
return dst
|
return dst
|
||||||
}
|
}
|
||||||
|
|
||||||
func fillContentsWithOwner(src []*data.ObjectInfo, encode string) []Object {
|
func fillContentsWithOwner(src []*data.ObjectInfo, encode string, md5Enabled bool) []Object {
|
||||||
return fillContents(src, encode, true)
|
return fillContents(src, encode, true, md5Enabled)
|
||||||
}
|
}
|
||||||
|
|
||||||
func fillContents(src []*data.ObjectInfo, encode string, fetchOwner bool) []Object {
|
func fillContents(src []*data.ObjectInfo, encode string, fetchOwner, md5Enabled bool) []Object {
|
||||||
var dst []Object
|
var dst []Object
|
||||||
for _, obj := range src {
|
for _, obj := range src {
|
||||||
res := Object{
|
res := Object{
|
||||||
Key: s3PathEncode(obj.Name, encode),
|
Key: s3PathEncode(obj.Name, encode),
|
||||||
Size: obj.Size,
|
Size: obj.Size,
|
||||||
LastModified: obj.Created.UTC().Format(time.RFC3339),
|
LastModified: obj.Created.UTC().Format(time.RFC3339),
|
||||||
ETag: obj.HashSum,
|
ETag: data.Quote(obj.ETag(md5Enabled)),
|
||||||
StorageClass: api.DefaultStorageClass,
|
StorageClass: api.DefaultStorageClass,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -292,7 +292,7 @@ func encodeListObjectVersionsToResponse(info *layer.ListObjectVersionsInfo, buck
|
||||||
},
|
},
|
||||||
Size: ver.ObjectInfo.Size,
|
Size: ver.ObjectInfo.Size,
|
||||||
VersionID: ver.Version(),
|
VersionID: ver.Version(),
|
||||||
ETag: ver.ObjectInfo.ETag(md5Enabled),
|
ETag: data.Quote(ver.ObjectInfo.ETag(md5Enabled)),
|
||||||
StorageClass: api.DefaultStorageClass,
|
StorageClass: api.DefaultStorageClass,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
|
@ -276,7 +276,7 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
|
|
||||||
s := &SendNotificationParams{
|
s := &SendNotificationParams{
|
||||||
Event: EventObjectCreatedPut,
|
Event: EventObjectCreatedPut,
|
||||||
NotificationInfo: data.NotificationInfoFromObject(objInfo),
|
NotificationInfo: data.NotificationInfoFromObject(objInfo, h.cfg.MD5Enabled()),
|
||||||
BktInfo: bktInfo,
|
BktInfo: bktInfo,
|
||||||
ReqInfo: reqInfo,
|
ReqInfo: reqInfo,
|
||||||
}
|
}
|
||||||
|
@ -327,7 +327,7 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
addSSECHeaders(w.Header(), r.Header)
|
addSSECHeaders(w.Header(), r.Header)
|
||||||
}
|
}
|
||||||
|
|
||||||
w.Header().Set(api.ETag, objInfo.ETag(h.cfg.MD5Enabled()))
|
w.Header().Set(api.ETag, data.Quote(objInfo.ETag(h.cfg.MD5Enabled())))
|
||||||
|
|
||||||
middleware.WriteSuccessResponseHeadersOnly(w)
|
middleware.WriteSuccessResponseHeadersOnly(w)
|
||||||
}
|
}
|
||||||
|
@ -522,7 +522,7 @@ func (h *handler) PostObject(w http.ResponseWriter, r *http.Request) {
|
||||||
|
|
||||||
s := &SendNotificationParams{
|
s := &SendNotificationParams{
|
||||||
Event: EventObjectCreatedPost,
|
Event: EventObjectCreatedPost,
|
||||||
NotificationInfo: data.NotificationInfoFromObject(objInfo),
|
NotificationInfo: data.NotificationInfoFromObject(objInfo, h.cfg.MD5Enabled()),
|
||||||
BktInfo: bktInfo,
|
BktInfo: bktInfo,
|
||||||
ReqInfo: reqInfo,
|
ReqInfo: reqInfo,
|
||||||
}
|
}
|
||||||
|
@ -591,7 +591,7 @@ func (h *handler) PostObject(w http.ResponseWriter, r *http.Request) {
|
||||||
resp := &PostResponse{
|
resp := &PostResponse{
|
||||||
Bucket: objInfo.Bucket,
|
Bucket: objInfo.Bucket,
|
||||||
Key: objInfo.Name,
|
Key: objInfo.Name,
|
||||||
ETag: objInfo.ETag(h.cfg.MD5Enabled()),
|
ETag: data.Quote(objInfo.ETag(h.cfg.MD5Enabled())),
|
||||||
}
|
}
|
||||||
w.WriteHeader(status)
|
w.WriteHeader(status)
|
||||||
if _, err = w.Write(middleware.EncodeResponse(resp)); err != nil {
|
if _, err = w.Write(middleware.EncodeResponse(resp)); err != nil {
|
||||||
|
@ -601,7 +601,7 @@ func (h *handler) PostObject(w http.ResponseWriter, r *http.Request) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
w.Header().Set(api.ETag, objInfo.HashSum)
|
w.Header().Set(api.ETag, data.Quote(objInfo.ETag(h.cfg.MD5Enabled())))
|
||||||
w.WriteHeader(status)
|
w.WriteHeader(status)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -19,6 +19,7 @@ import (
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
|
||||||
v4 "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth/signer/v4"
|
v4 "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth/signer/v4"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
s3errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
s3errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||||
|
@ -204,7 +205,7 @@ func TestPutObjectWithEnabledMD5(t *testing.T) {
|
||||||
md5Hash.Write(content)
|
md5Hash.Write(content)
|
||||||
w, r := prepareTestPayloadRequest(tc, bktName, objName, bytes.NewReader(content))
|
w, r := prepareTestPayloadRequest(tc, bktName, objName, bytes.NewReader(content))
|
||||||
tc.Handler().PutObjectHandler(w, r)
|
tc.Handler().PutObjectHandler(w, r)
|
||||||
require.Equal(t, hex.EncodeToString(md5Hash.Sum(nil)), w.Header().Get(api.ETag))
|
require.Equal(t, data.Quote(hex.EncodeToString(md5Hash.Sum(nil))), w.Header().Get(api.ETag))
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPutObjectWithStreamBodyAWSExample(t *testing.T) {
|
func TestPutObjectWithStreamBodyAWSExample(t *testing.T) {
|
||||||
|
|
|
@ -376,7 +376,7 @@ func (n *layer) CompleteMultipartUpload(ctx context.Context, p *CompleteMultipar
|
||||||
md5Hash := md5.New()
|
md5Hash := md5.New()
|
||||||
for i, part := range p.Parts {
|
for i, part := range p.Parts {
|
||||||
partInfo := partsInfo[part.PartNumber]
|
partInfo := partsInfo[part.PartNumber]
|
||||||
if partInfo == nil || (part.ETag != partInfo.ETag && part.ETag != partInfo.MD5) {
|
if partInfo == nil || data.UnQuote(part.ETag) != partInfo.GetETag(n.features.MD5Enabled()) {
|
||||||
return nil, nil, fmt.Errorf("%w: unknown part %d or etag mismatched", s3errors.GetAPIError(s3errors.ErrInvalidPart), part.PartNumber)
|
return nil, nil, fmt.Errorf("%w: unknown part %d or etag mismatched", s3errors.GetAPIError(s3errors.ErrInvalidPart), part.PartNumber)
|
||||||
}
|
}
|
||||||
delete(partsInfo, part.PartNumber)
|
delete(partsInfo, part.PartNumber)
|
||||||
|
@ -571,7 +571,7 @@ func (n *layer) ListParts(ctx context.Context, p *ListPartsParams) (*ListPartsIn
|
||||||
|
|
||||||
for _, partInfo := range partsInfo {
|
for _, partInfo := range partsInfo {
|
||||||
parts = append(parts, &Part{
|
parts = append(parts, &Part{
|
||||||
ETag: partInfo.ETag,
|
ETag: data.Quote(partInfo.GetETag(n.features.MD5Enabled())),
|
||||||
LastModified: partInfo.Created.UTC().Format(time.RFC3339),
|
LastModified: partInfo.Created.UTC().Format(time.RFC3339),
|
||||||
PartNumber: partInfo.Number,
|
PartNumber: partInfo.Number,
|
||||||
Size: partInfo.Size,
|
Size: partInfo.Size,
|
||||||
|
|
|
@ -9,6 +9,7 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
"github.com/go-chi/chi/v5"
|
"github.com/go-chi/chi/v5"
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
|
@ -293,7 +294,7 @@ func getSourceIP(r *http.Request) string {
|
||||||
if match := forRegex.FindStringSubmatch(fwd); len(match) > 1 {
|
if match := forRegex.FindStringSubmatch(fwd); len(match) > 1 {
|
||||||
// IPv6 addresses in Forwarded headers are quoted-strings. We strip
|
// IPv6 addresses in Forwarded headers are quoted-strings. We strip
|
||||||
// these quotes.
|
// these quotes.
|
||||||
addr = strings.Trim(match[1], `"`)
|
addr = data.UnQuote(match[1])
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue