[#221] Expand xmlns field ignore

Signed-off-by: Marina Biryukova <m.biryukova@yadro.com>

CHANGELOG.md

@@ -45,6 +45,7 @@ This document outlines major changes between releases.
 - Complete multipart upload doesn't unnecessary copy now. Thus, the total time of multipart upload was reduced by 2 times (#63)
 - Use gate key to form object owner (#175)
 - Apply placement policies and copies if there is at least one valid value (#168)
+- Generalise config param `use_default_xmlns_for_complete_multipart` to `use_default_xmlns` so that use default xmlns for all requests (#221)
 
 ### Removed
 - Drop `tree.service` param (now endpoints from `peers` section are used) (#133)

api/data/notifications.go

@@ -1,7 +1,10 @@
 package data
 
+import "encoding/xml"
+
 type (
 	NotificationConfiguration struct {
+		XMLName             xml.Name             `xml:"http://s3.amazonaws.com/doc/2006-03-01/ NotificationConfiguration" json:"-"`
 		QueueConfigurations []QueueConfiguration `xml:"QueueConfiguration" json:"QueueConfigurations"`
 		// Not supported topics
 		TopicConfigurations          []TopicConfiguration          `xml:"TopicConfiguration" json:"TopicConfigurations"`

api/handler/acl.go

@@ -6,7 +6,6 @@ import (
 	"crypto/elliptic"
 	"encoding/hex"
 	"encoding/json"
-	"encoding/xml"
 	stderrors "errors"
 	"fmt"
 	"net/http"
@@ -304,7 +303,7 @@ func (h *handler) PutBucketACLHandler(w http.ResponseWriter, r *http.Request) {
 			h.logAndSendError(w, "could not parse bucket acl", reqInfo, err)
 			return
 		}
-	} else if err = xml.NewDecoder(r.Body).Decode(list); err != nil {
+	} else if err = h.cfg.NewXMLDecoder(r.Body).Decode(list); err != nil {
 		h.logAndSendError(w, "could not parse bucket acl", reqInfo, errors.GetAPIError(errors.ErrMalformedXML))
 		return
 	}
@@ -441,7 +440,7 @@ func (h *handler) PutObjectACLHandler(w http.ResponseWriter, r *http.Request) {
 			h.logAndSendError(w, "could not parse bucket acl", reqInfo, err)
 			return
 		}
-	} else if err = xml.NewDecoder(r.Body).Decode(list); err != nil {
+	} else if err = h.cfg.NewXMLDecoder(r.Body).Decode(list); err != nil {
 		h.logAndSendError(w, "could not parse bucket acl", reqInfo, errors.GetAPIError(errors.ErrMalformedXML))
 		return
 	}

api/handler/api.go

@@ -35,7 +35,7 @@ type (
 		PlacementPolicy(string) (netmap.PlacementPolicy, bool)
 		CopiesNumbers(string) ([]uint32, bool)
 		DefaultCopiesNumbers() []uint32
-		NewCompleteMultipartDecoder(io.Reader) *xml.Decoder
+		NewXMLDecoder(io.Reader) *xml.Decoder
 		DefaultMaxAge() int
 		NotificatorEnabled() bool
 		ResolveZoneList() []string

api/handler/cors.go

@@ -52,6 +52,7 @@ func (h *handler) PutBucketCorsHandler(w http.ResponseWriter, r *http.Request) {
 	p := &layer.PutCORSParams{
 		BktInfo:    bktInfo,
 		Reader:     r.Body,
+		NewDecoder: h.cfg.NewXMLDecoder,
 	}
 
 	p.CopiesNumbers, err = h.pickCopiesNumbers(parseMetadata(r), bktInfo.LocationConstraint)

api/handler/delete.go

@@ -24,8 +24,9 @@ const maxObjectsToDelete = 1000
 
 // DeleteObjectsRequest -- xml carrying the object key names which should be deleted.
 type DeleteObjectsRequest struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Delete" json:"-"`
 	// Element to enable quiet mode for the request
-	Quiet bool
+	Quiet bool `xml:"Quiet,omitempty"`
 	// List of objects to be deleted
 	Objects []ObjectIdentifier `xml:"Object"`
 }
@@ -177,7 +178,7 @@ func (h *handler) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Re
 
 	// Unmarshal list of keys to be deleted.
 	requested := &DeleteObjectsRequest{}
-	if err := xml.NewDecoder(r.Body).Decode(requested); err != nil {
+	if err := h.cfg.NewXMLDecoder(r.Body).Decode(requested); err != nil {
 		h.logAndSendError(w, "couldn't decode body", reqInfo, errors.GetAPIError(errors.ErrMalformedXML))
 		return
 	}

api/handler/handlers_test.go

@@ -82,7 +82,7 @@ func (c *configMock) DefaultCopiesNumbers() []uint32 {
 	return c.defaultCopiesNumbers
 }
 
-func (c *configMock) NewCompleteMultipartDecoder(r io.Reader) *xml.Decoder {
+func (c *configMock) NewXMLDecoder(r io.Reader) *xml.Decoder {
 	return xml.NewDecoder(r)
 }
 

api/handler/locking.go

@@ -2,7 +2,6 @@ package handler
 
 import (
 	"context"
-	"encoding/xml"
 	"fmt"
 	"net/http"
 	"strconv"
@@ -42,7 +41,7 @@ func (h *handler) PutBucketObjectLockConfigHandler(w http.ResponseWriter, r *htt
 	}
 
 	lockingConf := &data.ObjectLockConfiguration{}
-	if err = xml.NewDecoder(r.Body).Decode(lockingConf); err != nil {
+	if err = h.cfg.NewXMLDecoder(r.Body).Decode(lockingConf); err != nil {
 		h.logAndSendError(w, "couldn't parse locking configuration", reqInfo, err)
 		return
 	}
@@ -122,7 +121,7 @@ func (h *handler) PutObjectLegalHoldHandler(w http.ResponseWriter, r *http.Reque
 	}
 
 	legalHold := &data.LegalHold{}
-	if err = xml.NewDecoder(r.Body).Decode(legalHold); err != nil {
+	if err = h.cfg.NewXMLDecoder(r.Body).Decode(legalHold); err != nil {
 		h.logAndSendError(w, "couldn't parse legal hold configuration", reqInfo, err)
 		return
 	}
@@ -210,7 +209,7 @@ func (h *handler) PutObjectRetentionHandler(w http.ResponseWriter, r *http.Reque
 	}
 
 	retention := &data.Retention{}
-	if err = xml.NewDecoder(r.Body).Decode(retention); err != nil {
+	if err = h.cfg.NewXMLDecoder(r.Body).Decode(retention); err != nil {
 		h.logAndSendError(w, "couldn't parse object retention", reqInfo, err)
 		return
 	}

api/handler/multipart_upload.go

@@ -406,7 +406,7 @@ func (h *handler) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.
 	)
 
 	reqBody := new(CompleteMultipartUpload)
-	if err = h.cfg.NewCompleteMultipartDecoder(r.Body).Decode(reqBody); err != nil {
+	if err = h.cfg.NewXMLDecoder(r.Body).Decode(reqBody); err != nil {
 		h.logAndSendError(w, "could not read complete multipart upload xml", reqInfo,
 			errors.GetAPIError(errors.ErrMalformedXML), additional...)
 		return

api/handler/notifications.go

@@ -2,7 +2,6 @@ package handler
 
 import (
 	"context"
-	"encoding/xml"
 	"fmt"
 	"net/http"
 	"strings"
@@ -26,11 +25,6 @@ type (
 		User             string
 		Time             time.Time
 	}
-
-	NotificationConfiguration struct {
-		XMLName                   xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ NotificationConfiguation"`
-		NotificationConfiguration data.NotificationConfiguration
-	}
 )
 
 const (
@@ -105,7 +99,7 @@ func (h *handler) PutBucketNotificationHandler(w http.ResponseWriter, r *http.Re
 	}
 
 	conf := &data.NotificationConfiguration{}
-	if err = xml.NewDecoder(r.Body).Decode(conf); err != nil {
+	if err = h.cfg.NewXMLDecoder(r.Body).Decode(conf); err != nil {
 		h.logAndSendError(w, "couldn't decode notification configuration", reqInfo, errors.GetAPIError(errors.ErrMalformedXML))
 		return
 	}

api/handler/put.go

@@ -433,7 +433,7 @@ func (h *handler) PostObject(w http.ResponseWriter, r *http.Request) {
 
 	if tagging := auth.MultipartFormValue(r, "tagging"); tagging != "" {
 		buffer := bytes.NewBufferString(tagging)
-		tagSet, err = readTagSet(buffer)
+		tagSet, err = h.readTagSet(buffer)
 		if err != nil {
 			h.logAndSendError(w, "could not read tag set", reqInfo, err)
 			return
@@ -742,7 +742,7 @@ func (h *handler) CreateBucketHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	createParams, err := parseLocationConstraint(r)
+	createParams, err := h.parseLocationConstraint(r)
 	if err != nil {
 		h.logAndSendError(w, "could not parse body", reqInfo, err)
 		return
@@ -859,13 +859,13 @@ func isAlphaNum(char int32) bool {
 	return 'a' <= char && char <= 'z' || '0' <= char && char <= '9'
 }
 
-func parseLocationConstraint(r *http.Request) (*createBucketParams, error) {
+func (h *handler) parseLocationConstraint(r *http.Request) (*createBucketParams, error) {
 	if r.ContentLength == 0 {
 		return new(createBucketParams), nil
 	}
 
 	params := new(createBucketParams)
-	if err := xml.NewDecoder(r.Body).Decode(params); err != nil {
+	if err := h.cfg.NewXMLDecoder(r.Body).Decode(params); err != nil {
 		return nil, errors.GetAPIError(errors.ErrMalformedXML)
 	}
 	return params, nil

api/handler/tagging.go

@@ -1,7 +1,6 @@
 package handler
 
 import (
-	"encoding/xml"
 	"io"
 	"net/http"
 	"sort"
@@ -29,7 +28,7 @@ func (h *handler) PutObjectTaggingHandler(w http.ResponseWriter, r *http.Request
 	ctx := r.Context()
 	reqInfo := middleware.GetReqInfo(ctx)
 
-	tagSet, err := readTagSet(r.Body)
+	tagSet, err := h.readTagSet(r.Body)
 	if err != nil {
 		h.logAndSendError(w, "could not read tag set", reqInfo, err)
 		return
@@ -153,7 +152,7 @@ func (h *handler) DeleteObjectTaggingHandler(w http.ResponseWriter, r *http.Requ
 func (h *handler) PutBucketTaggingHandler(w http.ResponseWriter, r *http.Request) {
 	reqInfo := middleware.GetReqInfo(r.Context())
 
-	tagSet, err := readTagSet(r.Body)
+	tagSet, err := h.readTagSet(r.Body)
 	if err != nil {
 		h.logAndSendError(w, "could not read tag set", reqInfo, err)
 		return
@@ -208,9 +207,9 @@ func (h *handler) DeleteBucketTaggingHandler(w http.ResponseWriter, r *http.Requ
 	w.WriteHeader(http.StatusNoContent)
 }
 
-func readTagSet(reader io.Reader) (map[string]string, error) {
+func (h *handler) readTagSet(reader io.Reader) (map[string]string, error) {
 	tagging := new(Tagging)
-	if err := xml.NewDecoder(reader).Decode(tagging); err != nil {
+	if err := h.cfg.NewXMLDecoder(reader).Decode(tagging); err != nil {
 		return nil, errors.GetAPIError(errors.ErrMalformedXML)
 	}
 

api/handler/versioning.go

@@ -1,7 +1,6 @@
 package handler
 
 import (
-	"encoding/xml"
 	"net/http"
 
 	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
@@ -14,7 +13,7 @@ func (h *handler) PutBucketVersioningHandler(w http.ResponseWriter, r *http.Requ
 	reqInfo := middleware.GetReqInfo(r.Context())
 
 	configuration := new(VersioningConfiguration)
-	if err := xml.NewDecoder(r.Body).Decode(configuration); err != nil {
+	if err := h.cfg.NewXMLDecoder(r.Body).Decode(configuration); err != nil {
 		h.logAndSendError(w, "couldn't decode versioning configuration", reqInfo, errors.GetAPIError(errors.ErrIllegalVersioningConfigurationException))
 		return
 	}

api/layer/cors.go

@@ -3,7 +3,6 @@ package layer
 import (
 	"bytes"
 	"context"
-	"encoding/xml"
 	errorsStd "errors"
 	"fmt"
 	"io"
@@ -25,7 +24,7 @@ func (n *layer) PutBucketCORS(ctx context.Context, p *PutCORSParams) error {
 		cors = &data.CORSConfiguration{}
 	)
 
-	if err := xml.NewDecoder(tee).Decode(cors); err != nil {
+	if err := p.NewDecoder(tee).Decode(cors); err != nil {
 		return fmt.Errorf("xml decode cors: %w", err)
 	}
 

api/layer/layer.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"crypto/ecdsa"
 	"crypto/rand"
+	"encoding/xml"
 	"fmt"
 	"io"
 	"net/url"
@@ -146,6 +147,7 @@ type (
 		BktInfo       *data.BucketInfo
 		Reader        io.Reader
 		CopiesNumbers []uint32
+		NewDecoder    func(io.Reader) *xml.Decoder
 	}
 
 	// CopyObjectParams stores object copy request parameters.

cmd/s3-gw/app.go

@@ -82,7 +82,7 @@ type (
 		regionMap                     map[string]netmap.PlacementPolicy
 		copiesNumbers                 map[string][]uint32
 		defaultCopiesNumbers          []uint32
-		defaultXMLNSForCompleteMultipart bool
+		defaultXMLNS                  bool
 		bypassContentEncodingInChunks bool
 		clientCut                     bool
 		maxBufferSizeForPut           uint64
@@ -175,7 +175,7 @@ func newAppSettings(log *Logger, v *viper.Viper) *appSettings {
 	settings := &appSettings{
 		logLevel:                   log.lvl,
 		maxClient:                  newMaxClients(v),
-		defaultXMLNSForCompleteMultipart: v.GetBool(cfgKludgeUseDefaultXMLNSForCompleteMultipartUpload),
+		defaultXMLNS:               v.GetBool(cfgKludgeUseDefaultXMLNS),
 		defaultMaxAge:              fetchDefaultMaxAge(v, log.logger),
 		notificatorEnabled:         v.GetBool(cfgEnableNATS),
 		completeMultipartKeepalive: v.GetDuration(cfgKludgeCompleteMultipartUploadKeepalive),
@@ -274,11 +274,11 @@ func (s *appSettings) DefaultCopiesNumbers() []uint32 {
 	return s.defaultCopiesNumbers
 }
 
-func (s *appSettings) NewCompleteMultipartDecoder(r io.Reader) *xml.Decoder {
+func (s *appSettings) NewXMLDecoder(r io.Reader) *xml.Decoder {
 	dec := xml.NewDecoder(r)
 
 	s.mu.RLock()
-	if s.defaultXMLNSForCompleteMultipart {
+	if s.defaultXMLNS {
 		dec.DefaultSpace = awsDefaultNamespace
 	}
 	s.mu.RUnlock()
@@ -286,9 +286,9 @@ func (s *appSettings) NewCompleteMultipartDecoder(r io.Reader) *xml.Decoder {
 	return dec
 }
 
-func (s *appSettings) useDefaultNamespaceForCompleteMultipart(useDefaultNamespace bool) {
+func (s *appSettings) useDefaultXMLNamespace(useDefaultNamespace bool) {
 	s.mu.Lock()
-	s.defaultXMLNSForCompleteMultipart = useDefaultNamespace
+	s.defaultXMLNS = useDefaultNamespace
 	s.mu.Unlock()
 }
 
@@ -600,7 +600,7 @@ func (a *App) updateSettings() {
 
 	a.settings.initPlacementPolicy(a.log, a.cfg)
 
-	a.settings.useDefaultNamespaceForCompleteMultipart(a.cfg.GetBool(cfgKludgeUseDefaultXMLNSForCompleteMultipartUpload))
+	a.settings.useDefaultXMLNamespace(a.cfg.GetBool(cfgKludgeUseDefaultXMLNS))
 	a.settings.setBypassContentEncodingInChunks(a.cfg.GetBool(cfgKludgeBypassContentEncodingCheckInChunks))
 	a.settings.setClientCut(a.cfg.GetBool(cfgClientCut))
 	a.settings.setBufferMaxSizeForPut(a.cfg.GetUint64(cfgBufferMaxSizeForPut))

cmd/s3-gw/app_settings.go

@@ -130,7 +130,7 @@ const ( // Settings.
 	cfgApplicationBuildTime = "app.build_time"
 
 	// Kludge.
-	cfgKludgeUseDefaultXMLNSForCompleteMultipartUpload = "kludge.use_default_xmlns_for_complete_multipart"
+	cfgKludgeUseDefaultXMLNS                    = "kludge.use_default_xmlns"
 	cfgKludgeCompleteMultipartUploadKeepalive   = "kludge.complete_multipart_keepalive"
 	cfgKludgeBypassContentEncodingCheckInChunks = "kludge.bypass_content_encoding_check_in_chunks"
 
@@ -534,7 +534,7 @@ func newSettings() *viper.Viper {
 	v.SetDefault(cfgBufferMaxSizeForPut, 1024*1024) // 1mb
 
 	// kludge
-	v.SetDefault(cfgKludgeUseDefaultXMLNSForCompleteMultipartUpload, false)
+	v.SetDefault(cfgKludgeUseDefaultXMLNS, false)
 	v.SetDefault(cfgKludgeCompleteMultipartUploadKeepalive, 10*time.Second)
 	v.SetDefault(cfgKludgeBypassContentEncodingCheckInChunks, false)
 

cmd/s3-gw/decoder_test.go

@@ -41,42 +41,42 @@ func TestDefaultNamespace(t *testing.T) {
 	}{
 		{
 			settings: &appSettings{
-				defaultXMLNSForCompleteMultipart: false,
+				defaultXMLNS: false,
 			},
 			input: xmlBodyWithNamespace,
 			err:   false,
 		},
 		{
 			settings: &appSettings{
-				defaultXMLNSForCompleteMultipart: false,
+				defaultXMLNS: false,
 			},
 			input: xmlBody,
 			err:   true,
 		},
 		{
 			settings: &appSettings{
-				defaultXMLNSForCompleteMultipart: false,
+				defaultXMLNS: false,
 			},
 			input: xmlBodyWithInvalidNamespace,
 			err:   true,
 		},
 		{
 			settings: &appSettings{
-				defaultXMLNSForCompleteMultipart: true,
+				defaultXMLNS: true,
 			},
 			input: xmlBodyWithNamespace,
 			err:   false,
 		},
 		{
 			settings: &appSettings{
-				defaultXMLNSForCompleteMultipart: true,
+				defaultXMLNS: true,
 			},
 			input: xmlBody,
 			err:   false,
 		},
 		{
 			settings: &appSettings{
-				defaultXMLNSForCompleteMultipart: true,
+				defaultXMLNS: true,
 			},
 			input: xmlBodyWithInvalidNamespace,
 			err:   true,
@@ -84,7 +84,7 @@ func TestDefaultNamespace(t *testing.T) {
 	} {
 		t.Run("", func(t *testing.T) {
 			model := new(handler.CompleteMultipartUpload)
-			err := tc.settings.NewCompleteMultipartDecoder(bytes.NewBufferString(tc.input)).Decode(model)
+			err := tc.settings.NewXMLDecoder(bytes.NewBufferString(tc.input)).Decode(model)
 			if tc.err {
 				require.Error(t, err)
 			} else {

config/config.env

@@ -138,8 +138,8 @@ S3_GW_ALLOWED_ACCESS_KEY_ID_PREFIXES=Ck9BHsgKcnwfCTUSFm6pxhoNS4cBqgN2NQ8zVgPjqZD
 S3_GW_RESOLVE_BUCKET_ALLOW=container
 # S3_GW_RESOLVE_BUCKET_DENY=
 
-# Enable using default xml namespace `http://s3.amazonaws.com/doc/2006-03-01/` when parse`CompleteMultipartUpload` xml body.
-S3_GW_KLUDGE_USE_DEFAULT_XMLNS_FOR_COMPLETE_MULTIPART=false
+# Enable using default xml namespace `http://s3.amazonaws.com/doc/2006-03-01/` when parse xml bodies.
+S3_GW_KLUDGE_USE_DEFAULT_XMLNS=false
 # Set timeout between whitespace transmissions during CompleteMultipartUpload processing.
 S3_GW_KLUDGE_COMPLETE_MULTIPART_KEEPALIVE=10s
 # Use this flag to be able to use chunked upload approach without having `aws-chunked` value in `Content-Encoding` header.

config/config.yaml

@@ -167,8 +167,8 @@ resolve_bucket:
   deny:
 
 kludge:
-  # Enable using default xml namespace `http://s3.amazonaws.com/doc/2006-03-01/` when parse`CompleteMultipartUpload` xml body.
-  use_default_xmlns_for_complete_multipart: false
+  # Enable using default xml namespace `http://s3.amazonaws.com/doc/2006-03-01/` when parse xml bodies.
+  use_default_xmlns: false
   # Set timeout between whitespace transmissions during CompleteMultipartUpload processing.
   complete_multipart_keepalive: 10s
   # Use this flag to be able to use chunked upload approach without having `aws-chunked` value in `Content-Encoding` header.

docs/configuration.md

@@ -540,14 +540,14 @@ Workarounds for non-standard use cases.
 
 ```yaml
 kludge:
-  use_default_xmlns_for_complete_multipart: false
+  use_default_xmlns: false
   complete_multipart_keepalive: 10s
   bypass_content_encoding_check_in_chunks: false
 ```
 
 | Parameter                                 | Type       | SIGHUP reload | Default value | Description                                                                                                                     |
-|--------------------------------------------|------------|---------------|---------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| `use_default_xmlns_for_complete_multipart` | `bool`     | yes           | false         | Enable using default xml namespace `http://s3.amazonaws.com/doc/2006-03-01/` when parse `CompleteMultipartUpload` xml body.                                                                      |
+|-------------------------------------------|------------|---------------|---------------|---------------------------------------------------------------------------------------------------------------------------------|
+| `use_default_xmlns`                       | `bool`     | yes           | false         | Enable using default xml namespace `http://s3.amazonaws.com/doc/2006-03-01/` when parse xml bodies.                             |
 | `complete_multipart_keepalive`            | `duration` | no            | 10s           | Set timeout between whitespace transmissions during CompleteMultipartUpload processing.                                         |
 | `bypass_content_encoding_check_in_chunks` | `bool`     | yes           | false         | Use this flag to be able to use [chunked upload approach](https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html) without having `aws-chunked` value in `Content-Encoding` header. |