acid-ant/frostfs-s3-gw
1
0
Fork 0
forked from TrueCloudLab/frostfs-s3-gw
Code Pull requests Activity

[#221] Expand xmlns field ignore

Browse source 
Signed-off-by: Marina Biryukova <m.biryukova@yadro.com>
This commit is contained in:
Marina Biryukova 2023-10-09 15:34:51 +03:00
parent 10a03faeb4
commit 298662df9d
22 changed files with 76 additions and 79 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
CHANGELOG.md
View file

@ -45,6 +45,7 @@ This document outlines major changes between releases.
- Complete multipart upload doesn't unnecessary copy now. Thus, the total time of multipart upload was reduced by 2 times (#63)
- Use gate key to form object owner (#175)
- Apply placement policies and copies if there is at least one valid value (#168)
- Generalise config param `use_default_xmlns_for_complete_multipart` to `use_default_xmlns` so that use default xmlns for all requests (#221)
### Removed
- Drop `tree.service` param (now endpoints from `peers` section are used) (#133)

3
api/data/notifications.go
View file

@ -1,7 +1,10 @@
package data
import "encoding/xml"
type (
NotificationConfiguration struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ NotificationConfiguration" json:"-"`
QueueConfigurations []QueueConfiguration `xml:"QueueConfiguration" json:"QueueConfigurations"`
// Not supported topics
TopicConfigurations []TopicConfiguration `xml:"TopicConfiguration" json:"TopicConfigurations"`

5
api/handler/acl.go
View file

@ -6,7 +6,6 @@ import (
"crypto/elliptic"
"encoding/hex"
"encoding/json"
"encoding/xml"
stderrors "errors"
"fmt"
"net/http"
@ -304,7 +303,7 @@ func (h *handler) PutBucketACLHandler(w http.ResponseWriter, r *http.Request) {
h.logAndSendError(w, "could not parse bucket acl", reqInfo, err)
return
}
} else if err = xml.NewDecoder(r.Body).Decode(list); err != nil {
} else if err = h.cfg.NewXMLDecoder(r.Body).Decode(list); err != nil {
h.logAndSendError(w, "could not parse bucket acl", reqInfo, errors.GetAPIError(errors.ErrMalformedXML))
return
}
@ -441,7 +440,7 @@ func (h *handler) PutObjectACLHandler(w http.ResponseWriter, r *http.Request) {
h.logAndSendError(w, "could not parse bucket acl", reqInfo, err)
return
}
} else if err = xml.NewDecoder(r.Body).Decode(list); err != nil {
} else if err = h.cfg.NewXMLDecoder(r.Body).Decode(list); err != nil {
h.logAndSendError(w, "could not parse bucket acl", reqInfo, errors.GetAPIError(errors.ErrMalformedXML))
return
}

2
api/handler/api.go
View file

@ -35,7 +35,7 @@ type (
PlacementPolicy(string) (netmap.PlacementPolicy, bool)
CopiesNumbers(string) ([]uint32, bool)
DefaultCopiesNumbers() []uint32
NewCompleteMultipartDecoder(io.Reader) *xml.Decoder
NewXMLDecoder(io.Reader) *xml.Decoder
DefaultMaxAge() int
NotificatorEnabled() bool
ResolveZoneList() []string

5
api/handler/cors.go
View file

@ -50,8 +50,9 @@ func (h *handler) PutBucketCorsHandler(w http.ResponseWriter, r *http.Request) {
}
p := &layer.PutCORSParams{
BktInfo: bktInfo,
Reader: r.Body,
BktInfo: bktInfo,
Reader: r.Body,
NewDecoder: h.cfg.NewXMLDecoder,
}
p.CopiesNumbers, err = h.pickCopiesNumbers(parseMetadata(r), bktInfo.LocationConstraint)

5
api/handler/delete.go
View file

@ -24,8 +24,9 @@ const maxObjectsToDelete = 1000
// DeleteObjectsRequest -- xml carrying the object key names which should be deleted.
type DeleteObjectsRequest struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Delete" json:"-"`
// Element to enable quiet mode for the request
Quiet bool
Quiet bool `xml:"Quiet,omitempty"`
// List of objects to be deleted
Objects []ObjectIdentifier `xml:"Object"`
}
@ -177,7 +178,7 @@ func (h *handler) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Re
// Unmarshal list of keys to be deleted.
requested := &DeleteObjectsRequest{}
if err := xml.NewDecoder(r.Body).Decode(requested); err != nil {
if err := h.cfg.NewXMLDecoder(r.Body).Decode(requested); err != nil {
h.logAndSendError(w, "couldn't decode body", reqInfo, errors.GetAPIError(errors.ErrMalformedXML))
return
}

2
api/handler/handlers_test.go
View file

@ -82,7 +82,7 @@ func (c *configMock) DefaultCopiesNumbers() []uint32 {
return c.defaultCopiesNumbers
}
func (c *configMock) NewCompleteMultipartDecoder(r io.Reader) *xml.Decoder {
func (c *configMock) NewXMLDecoder(r io.Reader) *xml.Decoder {
return xml.NewDecoder(r)
}

7
api/handler/locking.go
View file

@ -2,7 +2,6 @@ package handler
import (
"context"
"encoding/xml"
"fmt"
"net/http"
"strconv"
@ -42,7 +41,7 @@ func (h *handler) PutBucketObjectLockConfigHandler(w http.ResponseWriter, r *htt
}
lockingConf := &data.ObjectLockConfiguration{}
if err = xml.NewDecoder(r.Body).Decode(lockingConf); err != nil {
if err = h.cfg.NewXMLDecoder(r.Body).Decode(lockingConf); err != nil {
h.logAndSendError(w, "couldn't parse locking configuration", reqInfo, err)
return
}
@ -122,7 +121,7 @@ func (h *handler) PutObjectLegalHoldHandler(w http.ResponseWriter, r *http.Reque
}
legalHold := &data.LegalHold{}
if err = xml.NewDecoder(r.Body).Decode(legalHold); err != nil {
if err = h.cfg.NewXMLDecoder(r.Body).Decode(legalHold); err != nil {
h.logAndSendError(w, "couldn't parse legal hold configuration", reqInfo, err)
return
}
@ -210,7 +209,7 @@ func (h *handler) PutObjectRetentionHandler(w http.ResponseWriter, r *http.Reque
}
retention := &data.Retention{}
if err = xml.NewDecoder(r.Body).Decode(retention); err != nil {
if err = h.cfg.NewXMLDecoder(r.Body).Decode(retention); err != nil {
h.logAndSendError(w, "couldn't parse object retention", reqInfo, err)
return
}

2
api/handler/multipart_upload.go
View file

@ -406,7 +406,7 @@ func (h *handler) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.
)
reqBody := new(CompleteMultipartUpload)
if err = h.cfg.NewCompleteMultipartDecoder(r.Body).Decode(reqBody); err != nil {
if err = h.cfg.NewXMLDecoder(r.Body).Decode(reqBody); err != nil {
h.logAndSendError(w, "could not read complete multipart upload xml", reqInfo,
errors.GetAPIError(errors.ErrMalformedXML), additional...)
return

8
api/handler/notifications.go
View file

@ -2,7 +2,6 @@ package handler
import (
"context"
"encoding/xml"
"fmt"
"net/http"
"strings"
@ -26,11 +25,6 @@ type (
User string
Time time.Time
}
NotificationConfiguration struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ NotificationConfiguation"`
NotificationConfiguration data.NotificationConfiguration
}
)
const (
@ -105,7 +99,7 @@ func (h *handler) PutBucketNotificationHandler(w http.ResponseWriter, r *http.Re
}
conf := &data.NotificationConfiguration{}
if err = xml.NewDecoder(r.Body).Decode(conf); err != nil {
if err = h.cfg.NewXMLDecoder(r.Body).Decode(conf); err != nil {
h.logAndSendError(w, "couldn't decode notification configuration", reqInfo, errors.GetAPIError(errors.ErrMalformedXML))
return
}

8
api/handler/put.go
View file

@ -433,7 +433,7 @@ func (h *handler) PostObject(w http.ResponseWriter, r *http.Request) {
if tagging := auth.MultipartFormValue(r, "tagging"); tagging != "" {
buffer := bytes.NewBufferString(tagging)
tagSet, err = readTagSet(buffer)
tagSet, err = h.readTagSet(buffer)
if err != nil {
h.logAndSendError(w, "could not read tag set", reqInfo, err)
return
@ -742,7 +742,7 @@ func (h *handler) CreateBucketHandler(w http.ResponseWriter, r *http.Request) {
return
}
createParams, err := parseLocationConstraint(r)
createParams, err := h.parseLocationConstraint(r)
if err != nil {
h.logAndSendError(w, "could not parse body", reqInfo, err)
return
@ -859,13 +859,13 @@ func isAlphaNum(char int32) bool {
return 'a' <= char && char <= 'z' || '0' <= char && char <= '9'
}
func parseLocationConstraint(r *http.Request) (*createBucketParams, error) {
func (h *handler) parseLocationConstraint(r *http.Request) (*createBucketParams, error) {
if r.ContentLength == 0 {
return new(createBucketParams), nil
}
params := new(createBucketParams)
if err := xml.NewDecoder(r.Body).Decode(params); err != nil {
if err := h.cfg.NewXMLDecoder(r.Body).Decode(params); err != nil {
return nil, errors.GetAPIError(errors.ErrMalformedXML)
}
return params, nil

9
api/handler/tagging.go
View file

@ -1,7 +1,6 @@
package handler
import (
"encoding/xml"
"io"
"net/http"
"sort"
@ -29,7 +28,7 @@ func (h *handler) PutObjectTaggingHandler(w http.ResponseWriter, r *http.Request
ctx := r.Context()
reqInfo := middleware.GetReqInfo(ctx)
tagSet, err := readTagSet(r.Body)
tagSet, err := h.readTagSet(r.Body)
if err != nil {
h.logAndSendError(w, "could not read tag set", reqInfo, err)
return
@ -153,7 +152,7 @@ func (h *handler) DeleteObjectTaggingHandler(w http.ResponseWriter, r *http.Requ
func (h *handler) PutBucketTaggingHandler(w http.ResponseWriter, r *http.Request) {
reqInfo := middleware.GetReqInfo(r.Context())
tagSet, err := readTagSet(r.Body)
tagSet, err := h.readTagSet(r.Body)
if err != nil {
h.logAndSendError(w, "could not read tag set", reqInfo, err)
return
@ -208,9 +207,9 @@ func (h *handler) DeleteBucketTaggingHandler(w http.ResponseWriter, r *http.Requ
w.WriteHeader(http.StatusNoContent)
}
func readTagSet(reader io.Reader) (map[string]string, error) {
func (h *handler) readTagSet(reader io.Reader) (map[string]string, error) {
tagging := new(Tagging)
if err := xml.NewDecoder(reader).Decode(tagging); err != nil {
if err := h.cfg.NewXMLDecoder(reader).Decode(tagging); err != nil {
return nil, errors.GetAPIError(errors.ErrMalformedXML)
}

3
api/handler/versioning.go
View file

@ -1,7 +1,6 @@
package handler
import (
"encoding/xml"
"net/http"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
@ -14,7 +13,7 @@ func (h *handler) PutBucketVersioningHandler(w http.ResponseWriter, r *http.Requ
reqInfo := middleware.GetReqInfo(r.Context())
configuration := new(VersioningConfiguration)
if err := xml.NewDecoder(r.Body).Decode(configuration); err != nil {
if err := h.cfg.NewXMLDecoder(r.Body).Decode(configuration); err != nil {
h.logAndSendError(w, "couldn't decode versioning configuration", reqInfo, errors.GetAPIError(errors.ErrIllegalVersioningConfigurationException))
return
}

3
api/layer/cors.go
View file

@ -3,7 +3,6 @@ package layer
import (
"bytes"
"context"
"encoding/xml"
errorsStd "errors"
"fmt"
"io"
@ -25,7 +24,7 @@ func (n *layer) PutBucketCORS(ctx context.Context, p *PutCORSParams) error {
cors = &data.CORSConfiguration{}
)
if err := xml.NewDecoder(tee).Decode(cors); err != nil {
if err := p.NewDecoder(tee).Decode(cors); err != nil {
return fmt.Errorf("xml decode cors: %w", err)
}

2
api/layer/layer.go
View file

@ -4,6 +4,7 @@ import (
"context"
"crypto/ecdsa"
"crypto/rand"
"encoding/xml"
"fmt"
"io"
"net/url"
@ -146,6 +147,7 @@ type (
BktInfo *data.BucketInfo
Reader io.Reader
CopiesNumbers []uint32
NewDecoder func(io.Reader) *xml.Decoder
}
// CopyObjectParams stores object copy request parameters.

40
cmd/s3-gw/app.go
View file

@ -77,15 +77,15 @@ type (
isResolveListAllow bool // True if ResolveZoneList contains allowed zones
completeMultipartKeepalive time.Duration
mu sync.RWMutex
defaultPolicy netmap.PlacementPolicy
regionMap map[string]netmap.PlacementPolicy
copiesNumbers map[string][]uint32
defaultCopiesNumbers []uint32
defaultXMLNSForCompleteMultipart bool
bypassContentEncodingInChunks bool
clientCut bool
maxBufferSizeForPut uint64
mu sync.RWMutex
defaultPolicy netmap.PlacementPolicy
regionMap map[string]netmap.PlacementPolicy
copiesNumbers map[string][]uint32
defaultCopiesNumbers []uint32
defaultXMLNS bool
bypassContentEncodingInChunks bool
clientCut bool
maxBufferSizeForPut uint64
}
maxClientsConfig struct {
@ -173,12 +173,12 @@ func (a *App) initLayer(ctx context.Context) {
func newAppSettings(log *Logger, v *viper.Viper) *appSettings {
settings := &appSettings{
logLevel: log.lvl,
maxClient: newMaxClients(v),
defaultXMLNSForCompleteMultipart: v.GetBool(cfgKludgeUseDefaultXMLNSForCompleteMultipartUpload),
defaultMaxAge: fetchDefaultMaxAge(v, log.logger),
notificatorEnabled: v.GetBool(cfgEnableNATS),
completeMultipartKeepalive: v.GetDuration(cfgKludgeCompleteMultipartUploadKeepalive),
logLevel: log.lvl,
maxClient: newMaxClients(v),
defaultXMLNS: v.GetBool(cfgKludgeUseDefaultXMLNS),
defaultMaxAge: fetchDefaultMaxAge(v, log.logger),
notificatorEnabled: v.GetBool(cfgEnableNATS),
completeMultipartKeepalive: v.GetDuration(cfgKludgeCompleteMultipartUploadKeepalive),
}
settings.resolveZoneList = v.GetStringSlice(cfgResolveBucketAllow)
@ -274,11 +274,11 @@ func (s *appSettings) DefaultCopiesNumbers() []uint32 {
return s.defaultCopiesNumbers
}
func (s *appSettings) NewCompleteMultipartDecoder(r io.Reader) *xml.Decoder {
func (s *appSettings) NewXMLDecoder(r io.Reader) *xml.Decoder {
dec := xml.NewDecoder(r)
s.mu.RLock()
if s.defaultXMLNSForCompleteMultipart {
if s.defaultXMLNS {
dec.DefaultSpace = awsDefaultNamespace
}
s.mu.RUnlock()
@ -286,9 +286,9 @@ func (s *appSettings) NewCompleteMultipartDecoder(r io.Reader) *xml.Decoder {
return dec
}
func (s *appSettings) useDefaultNamespaceForCompleteMultipart(useDefaultNamespace bool) {
func (s *appSettings) useDefaultXMLNamespace(useDefaultNamespace bool) {
s.mu.Lock()
s.defaultXMLNSForCompleteMultipart = useDefaultNamespace
s.defaultXMLNS = useDefaultNamespace
s.mu.Unlock()
}
@ -600,7 +600,7 @@ func (a *App) updateSettings() {
a.settings.initPlacementPolicy(a.log, a.cfg)
a.settings.useDefaultNamespaceForCompleteMultipart(a.cfg.GetBool(cfgKludgeUseDefaultXMLNSForCompleteMultipartUpload))
a.settings.useDefaultXMLNamespace(a.cfg.GetBool(cfgKludgeUseDefaultXMLNS))
a.settings.setBypassContentEncodingInChunks(a.cfg.GetBool(cfgKludgeBypassContentEncodingCheckInChunks))
a.settings.setClientCut(a.cfg.GetBool(cfgClientCut))
a.settings.setBufferMaxSizeForPut(a.cfg.GetUint64(cfgBufferMaxSizeForPut))

8
cmd/s3-gw/app_settings.go
View file

@ -130,9 +130,9 @@ const ( // Settings.
cfgApplicationBuildTime = "app.build_time"
// Kludge.
cfgKludgeUseDefaultXMLNSForCompleteMultipartUpload = "kludge.use_default_xmlns_for_complete_multipart"
cfgKludgeCompleteMultipartUploadKeepalive = "kludge.complete_multipart_keepalive"
cfgKludgeBypassContentEncodingCheckInChunks = "kludge.bypass_content_encoding_check_in_chunks"
cfgKludgeUseDefaultXMLNS = "kludge.use_default_xmlns"
cfgKludgeCompleteMultipartUploadKeepalive = "kludge.complete_multipart_keepalive"
cfgKludgeBypassContentEncodingCheckInChunks = "kludge.bypass_content_encoding_check_in_chunks"
// Command line args.
cmdHelp = "help"
@ -534,7 +534,7 @@ func newSettings() *viper.Viper {
v.SetDefault(cfgBufferMaxSizeForPut, 1024*1024) // 1mb
// kludge
v.SetDefault(cfgKludgeUseDefaultXMLNSForCompleteMultipartUpload, false)
v.SetDefault(cfgKludgeUseDefaultXMLNS, false)
v.SetDefault(cfgKludgeCompleteMultipartUploadKeepalive, 10*time.Second)
v.SetDefault(cfgKludgeBypassContentEncodingCheckInChunks, false)

14
cmd/s3-gw/decoder_test.go
View file

@ -41,42 +41,42 @@ func TestDefaultNamespace(t *testing.T) {
}{
{
settings: &appSettings{
defaultXMLNSForCompleteMultipart: false,
defaultXMLNS: false,
},
input: xmlBodyWithNamespace,
err: false,
},
{
settings: &appSettings{
defaultXMLNSForCompleteMultipart: false,
defaultXMLNS: false,
},
input: xmlBody,
err: true,
},
{
settings: &appSettings{
defaultXMLNSForCompleteMultipart: false,
defaultXMLNS: false,
},
input: xmlBodyWithInvalidNamespace,
err: true,
},
{
settings: &appSettings{
defaultXMLNSForCompleteMultipart: true,
defaultXMLNS: true,
},
input: xmlBodyWithNamespace,
err: false,
},
{
settings: &appSettings{
defaultXMLNSForCompleteMultipart: true,
defaultXMLNS: true,
},
input: xmlBody,
err: false,
},
{
settings: &appSettings{
defaultXMLNSForCompleteMultipart: true,
defaultXMLNS: true,
},
input: xmlBodyWithInvalidNamespace,
err: true,
@ -84,7 +84,7 @@ func TestDefaultNamespace(t *testing.T) {
} {
t.Run("", func(t *testing.T) {
model := new(handler.CompleteMultipartUpload)
err := tc.settings.NewCompleteMultipartDecoder(bytes.NewBufferString(tc.input)).Decode(model)
err := tc.settings.NewXMLDecoder(bytes.NewBufferString(tc.input)).Decode(model)
if tc.err {
require.Error(t, err)
} else {

4
config/config.env
View file

@ -138,8 +138,8 @@ S3_GW_ALLOWED_ACCESS_KEY_ID_PREFIXES=Ck9BHsgKcnwfCTUSFm6pxhoNS4cBqgN2NQ8zVgPjqZD
S3_GW_RESOLVE_BUCKET_ALLOW=container
# S3_GW_RESOLVE_BUCKET_DENY=
# Enable using default xml namespace `http://s3.amazonaws.com/doc/2006-03-01/` when parse`CompleteMultipartUpload` xml body.
S3_GW_KLUDGE_USE_DEFAULT_XMLNS_FOR_COMPLETE_MULTIPART=false
# Enable using default xml namespace `http://s3.amazonaws.com/doc/2006-03-01/` when parse xml bodies.
S3_GW_KLUDGE_USE_DEFAULT_XMLNS=false
# Set timeout between whitespace transmissions during CompleteMultipartUpload processing.
S3_GW_KLUDGE_COMPLETE_MULTIPART_KEEPALIVE=10s
# Use this flag to be able to use chunked upload approach without having `aws-chunked` value in `Content-Encoding` header.

4
config/config.yaml
View file

@ -167,8 +167,8 @@ resolve_bucket:
deny:
kludge:
# Enable using default xml namespace `http://s3.amazonaws.com/doc/2006-03-01/` when parse`CompleteMultipartUpload` xml body.
use_default_xmlns_for_complete_multipart: false
# Enable using default xml namespace `http://s3.amazonaws.com/doc/2006-03-01/` when parse xml bodies.
use_default_xmlns: false
# Set timeout between whitespace transmissions during CompleteMultipartUpload processing.
complete_multipart_keepalive: 10s
# Use this flag to be able to use chunked upload approach without having `aws-chunked` value in `Content-Encoding` header.

12
docs/configuration.md
View file

@ -540,16 +540,16 @@ Workarounds for non-standard use cases.
```yaml
kludge:
use_default_xmlns_for_complete_multipart: false
use_default_xmlns: false
complete_multipart_keepalive: 10s
bypass_content_encoding_check_in_chunks: false
```
| Parameter | Type | SIGHUP reload | Default value | Description |
|--------------------------------------------|------------|---------------|---------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `use_default_xmlns_for_complete_multipart` | `bool` | yes | false | Enable using default xml namespace `http://s3.amazonaws.com/doc/2006-03-01/` when parse `CompleteMultipartUpload` xml body. |
| `complete_multipart_keepalive` | `duration` | no | 10s | Set timeout between whitespace transmissions during CompleteMultipartUpload processing. |
| `bypass_content_encoding_check_in_chunks` | `bool` | yes | false | Use this flag to be able to use [chunked upload approach](https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html) without having `aws-chunked` value in `Content-Encoding` header. |
| Parameter | Type | SIGHUP reload | Default value | Description |
|-------------------------------------------|------------|---------------|---------------|---------------------------------------------------------------------------------------------------------------------------------|
| `use_default_xmlns` | `bool` | yes | false | Enable using default xml namespace `http://s3.amazonaws.com/doc/2006-03-01/` when parse xml bodies. |
| `complete_multipart_keepalive` | `duration` | no | 10s | Set timeout between whitespace transmissions during CompleteMultipartUpload processing. |
| `bypass_content_encoding_check_in_chunks` | `bool` | yes | false | Use this flag to be able to use [chunked upload approach](https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html) without having `aws-chunked` value in `Content-Encoding` header. |
# `runtime` section
Contains runtime parameters.